From 0297b36bda332fe21ab7d4c4c549cc68ccd344bc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 13:58:56 +0900 Subject: [PATCH 1/5] Fix reading more than read_size. --- msgpack/_msgpack.pyx | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7ff0cff..15bf5a7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -278,8 +278,8 @@ cdef class Unpacker(object): `unicode_errors` is used for decoding bytes. - `max_buffer_size` limits size of data waiting unpacked. 0 means unlimited - (default). + `max_buffer_size` limits size of data waiting unpacked. + 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. You shoud set this parameter when unpacking data from untrasted source. @@ -332,11 +332,11 @@ cdef class Unpacker(object): raise ValueError("`file_like.read` must be a callable.") if not max_buffer_size: max_buffer_size = INT_MAX + if read_size > max_buffer_size: + raise ValueError("read_size should be less or equal to max_buffer_size") if not read_size: read_size = min(max_buffer_size, 1024**2) self.max_buffer_size = max_buffer_size - if read_size > max_buffer_size: - raise ValueError("read_size should be less or equal to max_buffer_size") self.read_size = read_size self.buf = malloc(read_size) if self.buf == NULL: @@ -419,18 +419,15 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - # prepare self.buf from file_like - cdef fill_buffer(self): - if self.file_like is not None: - next_bytes = self.file_like_read( - max(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) - if next_bytes: - self.append_buffer(PyBytes_AsString(next_bytes), - PyBytes_Size(next_bytes)) - else: - self.file_like = None + cdef read_from_file(self): + next_bytes = self.file_like_read( + min(self.read_size, + self.max_buffer_size - (self.buf_tail - self.buf_head) + )) + if next_bytes: + self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) + else: + self.file_like = None cpdef unpack(self): """unpack one object""" @@ -443,7 +440,7 @@ cdef class Unpacker(object): return o elif ret == 0: if self.file_like is not None: - self.fill_buffer() + self.read_from_file() continue raise StopIteration("No more unpack data.") else: From 397d772e110ad7000e0952d41c74d8efd322f01f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:08:34 +0900 Subject: [PATCH 2/5] Rename use_float to use_single_float. --- msgpack/_msgpack.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index aef1228..af005cb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -65,8 +65,8 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_float=False): - self.use_float = use_float + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): + self.use_float = use_single_float if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") From 51335bbee4502ac3af81363a10ef6718439377d1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:15:30 +0900 Subject: [PATCH 3/5] packb supports use_single_float option. --- ChangeLog.rst | 9 ++++++++- msgpack/_msgpack.pyx | 5 +++-- test/test_pack.py | 5 +++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 67eac68..4fd5cbc 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,11 @@ 0.2.2 ======= -:release date: NOT RELEASED YET +:release date: 2012-09-21 + +Changes +------- +* Add ``use_single_float`` option to ``Packer``. When it is true, packs float + object in single precision format. Bugs fixed ----------- @@ -8,6 +13,8 @@ Bugs fixed ``unpack()`` doesn't control gc now instead of restoring gc state collectly. User can control gc state when gc cause performance issue. +* ``Unpacker``'s ``read_size`` option didn't used. + 0.2.1 ======= :release date: 2012-08-20 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index af005cb..c9f5e31 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -177,10 +177,11 @@ def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) -def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): +def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): """ pack o and return packed bytes.""" - packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) + packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, + use_single_float=use_single_float) return packer.pack(o) diff --git a/test/test_pack.py b/test/test_pack.py index 898cdb9..85d11a0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -2,6 +2,7 @@ # coding: utf-8 import six +import struct from nose import main from nose.tools import * from nose.plugins.skip import SkipTest @@ -86,5 +87,9 @@ def testDecodeBinary(): re = unpackb(packb("abc"), encoding=None) assert_equal(re, b"abc") +def testPackFloat(): + assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) + assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) + if __name__ == '__main__': main() From be405ec5cfe98a42283055baa901ec31de2632ba Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:16:40 +0900 Subject: [PATCH 4/5] 0.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2a20e16..ae72221 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 1, 'dev1') +version = (0, 2, 2) import os import sys From 5b66edaa156c43793b6f68013a738f545885b8d6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:17:34 +0900 Subject: [PATCH 5/5] 0.2.2 (again) --- msgpack/_version.py | 2 +- setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 68ae707..f343b7a 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 1) +version = (0, 2, 2) diff --git a/setup.py b/setup.py index ae72221..86b0b34 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 2) - import os import sys import shutil