mirror of
https://github.com/msgpack/msgpack-python.git
synced 2025-10-19 12:03:15 +00:00
Compare commits
8 commits
Author | SHA1 | Date | |
---|---|---|---|
![]() |
c1ecd23dbf | ||
![]() |
af45640970 | ||
![]() |
c2546eabc4 | ||
![]() |
ef4f83df16 | ||
![]() |
19b5d33ded | ||
![]() |
0f3c4be465 | ||
![]() |
c2a9f1fda5 | ||
![]() |
d9873dab04 |
12 changed files with 90 additions and 71 deletions
18
.github/workflows/test.yml
vendored
18
.github/workflows/test.yml
vendored
|
@ -9,18 +9,20 @@ jobs:
|
|||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: ["ubuntu-latest", "windows-latest", "macos-latest"]
|
||||
py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
|
||||
|
||||
os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"]
|
||||
py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"]
|
||||
exclude:
|
||||
- os: windows-11-arm
|
||||
py: "3.10"
|
||||
runs-on: ${{ matrix.os }}
|
||||
name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.py }}
|
||||
allow-prereleases: true
|
||||
|
@ -29,8 +31,7 @@ jobs:
|
|||
- name: Prepare
|
||||
shell: bash
|
||||
run: |
|
||||
pip install -U pip
|
||||
pip install -r requirements.txt pytest
|
||||
python -m pip install -r requirements.txt pytest
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
|
@ -51,8 +52,7 @@ jobs:
|
|||
- name: build packages
|
||||
shell: bash
|
||||
run: |
|
||||
pip install build
|
||||
python -m build
|
||||
python -m build -nv
|
||||
|
||||
- name: upload packages
|
||||
uses: actions/upload-artifact@v4
|
||||
|
|
10
.github/workflows/wheel.yml
vendored
10
.github/workflows/wheel.yml
vendored
|
@ -12,13 +12,13 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
# macos-13 is for intel
|
||||
os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"]
|
||||
os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"]
|
||||
runs-on: ${{ matrix.os }}
|
||||
name: Build wheels on ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: "pip"
|
||||
|
@ -29,11 +29,11 @@ jobs:
|
|||
make cython
|
||||
|
||||
- name: Build
|
||||
uses: pypa/cibuildwheel@v2.23.3
|
||||
uses: pypa/cibuildwheel@v3.2.0
|
||||
env:
|
||||
CIBW_TEST_REQUIRES: "pytest"
|
||||
CIBW_TEST_COMMAND: "pytest {package}/test"
|
||||
CIBW_SKIP: "pp* cp38-macosx_*"
|
||||
CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64"
|
||||
|
||||
- name: Build sdist
|
||||
if: runner.os == 'Linux' && runner.arch == 'X64'
|
||||
|
|
|
@ -1,3 +1,16 @@
|
|||
1.1.2
|
||||
=====
|
||||
|
||||
Release Date: 2025-10-08
|
||||
|
||||
This release does not change source code. It updates only building wheels:
|
||||
|
||||
* Update Cython to v3.1.4
|
||||
* Update cibuildwheel to v3.2.0
|
||||
* Drop Python 3.8
|
||||
* Add Python 3.14
|
||||
* Add windows-arm
|
||||
|
||||
1.1.1
|
||||
=====
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
include setup.py
|
||||
include COPYING
|
||||
include README.md
|
||||
recursive-include msgpack *.h *.c *.pyx *.cpp
|
||||
recursive-include msgpack *.h *.c *.pyx
|
||||
recursive-include test *.py
|
||||
|
|
64
README.md
64
README.md
|
@ -3,7 +3,7 @@
|
|||
[](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml)
|
||||
[](https://msgpack-python.readthedocs.io/en/latest/?badge=latest)
|
||||
|
||||
## What's this
|
||||
## What is this?
|
||||
|
||||
[MessagePack](https://msgpack.org/) is an efficient binary serialization format.
|
||||
It lets you exchange data among multiple languages like JSON.
|
||||
|
@ -25,9 +25,9 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy.
|
|||
|
||||
### Windows
|
||||
|
||||
When you can't use a binary distribution, you need to install Visual Studio
|
||||
or Windows SDK on Windows.
|
||||
Without extension, using pure Python implementation on CPython runs slowly.
|
||||
If you can't use a binary distribution, you need to install Visual Studio
|
||||
or the Windows SDK on Windows.
|
||||
Without the extension, the pure Python implementation on CPython runs slowly.
|
||||
|
||||
|
||||
## How to use
|
||||
|
@ -35,11 +35,11 @@ Without extension, using pure Python implementation on CPython runs slowly.
|
|||
### One-shot pack & unpack
|
||||
|
||||
Use `packb` for packing and `unpackb` for unpacking.
|
||||
msgpack provides `dumps` and `loads` as an alias for compatibility with
|
||||
msgpack provides `dumps` and `loads` as aliases for compatibility with
|
||||
`json` and `pickle`.
|
||||
|
||||
`pack` and `dump` packs to a file-like object.
|
||||
`unpack` and `load` unpacks from a file-like object.
|
||||
`pack` and `dump` pack to a file-like object.
|
||||
`unpack` and `load` unpack from a file-like object.
|
||||
|
||||
```pycon
|
||||
>>> import msgpack
|
||||
|
@ -73,7 +73,7 @@ for unpacked in unpacker:
|
|||
```
|
||||
|
||||
|
||||
### Packing/unpacking of custom data type
|
||||
### Packing/unpacking of custom data types
|
||||
|
||||
It is also possible to pack/unpack custom data types. Here is an example for
|
||||
`datetime.datetime`.
|
||||
|
@ -140,8 +140,8 @@ True
|
|||
### Advanced unpacking control
|
||||
|
||||
As an alternative to iteration, `Unpacker` objects provide `unpack`,
|
||||
`skip`, `read_array_header` and `read_map_header` methods. The former two
|
||||
read an entire message from the stream, respectively de-serialising and returning
|
||||
`skip`, `read_array_header`, and `read_map_header` methods. The former two
|
||||
read an entire message from the stream, respectively deserializing and returning
|
||||
the result, or ignoring it. The latter two methods return the number of elements
|
||||
in the upcoming container, so that each element in an array, or key-value pair
|
||||
in a map, can be unpacked or skipped individually.
|
||||
|
@ -149,7 +149,7 @@ in a map, can be unpacked or skipped individually.
|
|||
|
||||
## Notes
|
||||
|
||||
### string and binary type in old msgpack spec
|
||||
### String and binary types in the old MessagePack spec
|
||||
|
||||
Early versions of msgpack didn't distinguish string and binary types.
|
||||
The type for representing both string and binary types was named **raw**.
|
||||
|
@ -167,7 +167,7 @@ and `raw=True` options.
|
|||
|
||||
### ext type
|
||||
|
||||
To use the **ext** type, pass `msgpack.ExtType` object to packer.
|
||||
To use the **ext** type, pass a `msgpack.ExtType` object to the packer.
|
||||
|
||||
```pycon
|
||||
>>> import msgpack
|
||||
|
@ -181,26 +181,26 @@ You can use it with `default` and `ext_hook`. See below.
|
|||
|
||||
### Security
|
||||
|
||||
To unpacking data received from unreliable source, msgpack provides
|
||||
When unpacking data received from an unreliable source, msgpack provides
|
||||
two security options.
|
||||
|
||||
`max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size.
|
||||
It is used to limit the preallocated list size too.
|
||||
It is also used to limit preallocated list sizes.
|
||||
|
||||
`strict_map_key` (default: `True`) limits the type of map keys to bytes and str.
|
||||
While msgpack spec doesn't limit the types of the map keys,
|
||||
there is a risk of the hashdos.
|
||||
While the MessagePack spec doesn't limit map key types,
|
||||
there is a risk of a hash DoS.
|
||||
If you need to support other types for map keys, use `strict_map_key=False`.
|
||||
|
||||
|
||||
### Performance tips
|
||||
|
||||
CPython's GC starts when growing allocated object.
|
||||
This means unpacking may cause useless GC.
|
||||
You can use `gc.disable()` when unpacking large message.
|
||||
CPython's GC starts when the number of allocated objects grows.
|
||||
This means unpacking may trigger unnecessary GC.
|
||||
You can use `gc.disable()` when unpacking a large message.
|
||||
|
||||
List is the default sequence type of Python.
|
||||
But tuple is lighter than list.
|
||||
A list is the default sequence type in Python.
|
||||
However, a tuple is lighter than a list.
|
||||
You can use `use_list=False` while unpacking when performance is important.
|
||||
|
||||
|
||||
|
@ -208,7 +208,7 @@ You can use `use_list=False` while unpacking when performance is important.
|
|||
|
||||
### msgpack 0.5
|
||||
|
||||
Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5.
|
||||
The package name on PyPI was changed from `msgpack-python` to `msgpack` in 0.5.
|
||||
|
||||
When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before
|
||||
`pip install -U msgpack`.
|
||||
|
@ -218,25 +218,25 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be
|
|||
|
||||
* Python 2 support
|
||||
|
||||
* The extension module does not support Python 2 anymore.
|
||||
* The extension module no longer supports Python 2.
|
||||
The pure Python implementation (`msgpack.fallback`) is used for Python 2.
|
||||
|
||||
* msgpack 1.0.6 drops official support of Python 2.7, as pip and
|
||||
GitHub Action (setup-python) no longer support Python 2.7.
|
||||
GitHub Action "setup-python" no longer supports Python 2.7.
|
||||
|
||||
* Packer
|
||||
|
||||
* Packer uses `use_bin_type=True` by default.
|
||||
Bytes are encoded in bin type in msgpack.
|
||||
* The `encoding` option is removed. UTF-8 is used always.
|
||||
Bytes are encoded in the bin type in MessagePack.
|
||||
* The `encoding` option is removed. UTF-8 is always used.
|
||||
|
||||
* Unpacker
|
||||
|
||||
* Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string
|
||||
and decode them to Python str (unicode) object.
|
||||
* Unpacker uses `raw=False` by default. It assumes str values are valid UTF-8 strings
|
||||
and decodes them to Python str (Unicode) objects.
|
||||
* `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str).
|
||||
* Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack.
|
||||
* The default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attacks.
|
||||
You need to pass `max_buffer_size=0` if you have large but safe data.
|
||||
* Default value of `strict_map_key` is changed to True to avoid hashdos.
|
||||
You need to pass `strict_map_key=False` if you have data which contain map keys
|
||||
which type is not bytes or str.
|
||||
* The default value of `strict_map_key` is changed to True to avoid hash DoS.
|
||||
You need to pass `strict_map_key=False` if you have data that contain map keys
|
||||
whose type is neither bytes nor str.
|
||||
|
|
|
@ -4,8 +4,8 @@ import os
|
|||
from .exceptions import * # noqa: F403
|
||||
from .ext import ExtType, Timestamp
|
||||
|
||||
version = (1, 1, 1)
|
||||
__version__ = "1.1.1"
|
||||
version = (1, 1, 2)
|
||||
__version__ = "1.1.2"
|
||||
|
||||
|
||||
if os.environ.get("MSGPACK_PUREPYTHON"):
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
# coding: utf-8
|
||||
#cython: embedsignature=True, c_string_encoding=ascii, language_level=3
|
||||
#cython: freethreading_compatible = True
|
||||
import cython
|
||||
from cpython.datetime cimport import_datetime, datetime_new
|
||||
import_datetime()
|
||||
|
||||
|
|
|
@ -1,5 +1,3 @@
|
|||
# coding: utf-8
|
||||
|
||||
from cpython cimport *
|
||||
from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
|
||||
from cpython.datetime cimport (
|
||||
|
@ -129,6 +127,7 @@ cdef class Packer:
|
|||
if self.exports > 0:
|
||||
raise BufferError("Existing exports of data: Packer cannot be changed")
|
||||
|
||||
@cython.critical_section
|
||||
def __init__(self, *, default=None,
|
||||
bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
|
||||
bint strict_types=False, bint datetime=False, unicode_errors=None,
|
||||
|
@ -269,6 +268,7 @@ cdef class Packer:
|
|||
return ret
|
||||
return self._pack_inner(o, 0, nest_limit)
|
||||
|
||||
@cython.critical_section
|
||||
def pack(self, object obj):
|
||||
cdef int ret
|
||||
self._check_exports()
|
||||
|
@ -284,6 +284,7 @@ cdef class Packer:
|
|||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
@cython.critical_section
|
||||
def pack_ext_type(self, typecode, data):
|
||||
self._check_exports()
|
||||
if len(data) > ITEM_LIMIT:
|
||||
|
@ -291,6 +292,7 @@ cdef class Packer:
|
|||
msgpack_pack_ext(&self.pk, typecode, len(data))
|
||||
msgpack_pack_raw_body(&self.pk, data, len(data))
|
||||
|
||||
@cython.critical_section
|
||||
def pack_array_header(self, long long size):
|
||||
self._check_exports()
|
||||
if size > ITEM_LIMIT:
|
||||
|
@ -301,6 +303,7 @@ cdef class Packer:
|
|||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
@cython.critical_section
|
||||
def pack_map_header(self, long long size):
|
||||
self._check_exports()
|
||||
if size > ITEM_LIMIT:
|
||||
|
@ -311,6 +314,7 @@ cdef class Packer:
|
|||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
@cython.critical_section
|
||||
def pack_map_pairs(self, object pairs):
|
||||
"""
|
||||
Pack *pairs* as msgpack map type.
|
||||
|
@ -331,6 +335,7 @@ cdef class Packer:
|
|||
self.pk.length = 0
|
||||
return buf
|
||||
|
||||
@cython.critical_section
|
||||
def reset(self):
|
||||
"""Reset internal buffer.
|
||||
|
||||
|
@ -339,6 +344,7 @@ cdef class Packer:
|
|||
self._check_exports()
|
||||
self.pk.length = 0
|
||||
|
||||
@cython.critical_section
|
||||
def bytes(self):
|
||||
"""Return internal buffer contents as bytes object"""
|
||||
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
|
||||
|
|
|
@ -1,5 +1,3 @@
|
|||
# coding: utf-8
|
||||
|
||||
from cpython cimport *
|
||||
cdef extern from "Python.h":
|
||||
ctypedef struct PyObject
|
||||
|
@ -324,6 +322,7 @@ cdef class Unpacker:
|
|||
PyMem_Free(self.buf)
|
||||
self.buf = NULL
|
||||
|
||||
@cython.critical_section
|
||||
def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
|
||||
bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
|
||||
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
|
||||
|
@ -384,6 +383,7 @@ cdef class Unpacker:
|
|||
max_str_len, max_bin_len, max_array_len,
|
||||
max_map_len, max_ext_len)
|
||||
|
||||
@cython.critical_section
|
||||
def feed(self, object next_bytes):
|
||||
"""Append `next_bytes` to internal buffer."""
|
||||
cdef Py_buffer pybuff
|
||||
|
@ -484,6 +484,7 @@ cdef class Unpacker:
|
|||
else:
|
||||
raise ValueError("Unpack failed: error = %d" % (ret,))
|
||||
|
||||
@cython.critical_section
|
||||
def read_bytes(self, Py_ssize_t nbytes):
|
||||
"""Read a specified number of raw bytes from the stream"""
|
||||
cdef Py_ssize_t nread
|
||||
|
@ -496,6 +497,7 @@ cdef class Unpacker:
|
|||
self.stream_offset += nread
|
||||
return ret
|
||||
|
||||
@cython.critical_section
|
||||
def unpack(self):
|
||||
"""Unpack one object
|
||||
|
||||
|
@ -503,6 +505,7 @@ cdef class Unpacker:
|
|||
"""
|
||||
return self._unpack(unpack_construct)
|
||||
|
||||
@cython.critical_section
|
||||
def skip(self):
|
||||
"""Read and ignore one object, returning None
|
||||
|
||||
|
@ -510,6 +513,7 @@ cdef class Unpacker:
|
|||
"""
|
||||
return self._unpack(unpack_skip)
|
||||
|
||||
@cython.critical_section
|
||||
def read_array_header(self):
|
||||
"""assuming the next object is an array, return its size n, such that
|
||||
the next n unpack() calls will iterate over its contents.
|
||||
|
@ -518,6 +522,7 @@ cdef class Unpacker:
|
|||
"""
|
||||
return self._unpack(read_array_header)
|
||||
|
||||
@cython.critical_section
|
||||
def read_map_header(self):
|
||||
"""assuming the next object is a map, return its size n, such that the
|
||||
next n * 2 unpack() calls will iterate over its key-value pairs.
|
||||
|
@ -526,6 +531,7 @@ cdef class Unpacker:
|
|||
"""
|
||||
return self._unpack(read_map_header)
|
||||
|
||||
@cython.critical_section
|
||||
def tell(self):
|
||||
"""Returns the current position of the Unpacker in bytes, i.e., the
|
||||
number of bytes that were read from the input, also the starting
|
||||
|
@ -536,6 +542,7 @@ cdef class Unpacker:
|
|||
def __iter__(self):
|
||||
return self
|
||||
|
||||
@cython.critical_section
|
||||
def __next__(self):
|
||||
return self._unpack(unpack_construct, 1)
|
||||
|
||||
|
|
|
@ -1,31 +1,21 @@
|
|||
[build-system]
|
||||
# 75.3.0 is the latest version supporting Python 3.8
|
||||
requires = ["setuptools >= 75.3.0"]
|
||||
requires = ["setuptools >= 78.1.1"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "msgpack"
|
||||
dynamic = ["version"]
|
||||
# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility.
|
||||
# https://github.com/msgpack/msgpack-python/pull/637
|
||||
license = {text="Apache 2.0"}
|
||||
license = "Apache-2.0"
|
||||
authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
|
||||
description = "MessagePack serializer"
|
||||
readme = "README.md"
|
||||
keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"]
|
||||
requires-python = ">=3.8"
|
||||
requires-python = ">=3.10"
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Operating System :: OS Independent",
|
||||
"Topic :: File Formats",
|
||||
"Intended Audience :: Developers",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Python :: Implementation :: PyPy",
|
||||
]
|
||||
|
@ -46,7 +36,7 @@ version = {attr = "msgpack.__version__"}
|
|||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py38"
|
||||
target-version = "py310"
|
||||
lint.select = [
|
||||
"E", # pycodestyle
|
||||
"F", # Pyflakes
|
||||
|
|
|
@ -1 +1,3 @@
|
|||
Cython~=3.1.1
|
||||
Cython==3.1.4
|
||||
setuptools==78.1.1
|
||||
build
|
||||
|
|
|
@ -17,7 +17,7 @@ def test_unpack_bytearray():
|
|||
obj = unpackb(buf, use_list=1)
|
||||
assert [b"foo", b"bar"] == obj
|
||||
expected_type = bytes
|
||||
assert all(type(s) == expected_type for s in obj)
|
||||
assert all(type(s) is expected_type for s in obj)
|
||||
|
||||
|
||||
def test_unpack_memoryview():
|
||||
|
@ -26,7 +26,7 @@ def test_unpack_memoryview():
|
|||
obj = unpackb(view, use_list=1)
|
||||
assert [b"foo", b"bar"] == obj
|
||||
expected_type = bytes
|
||||
assert all(type(s) == expected_type for s in obj)
|
||||
assert all(type(s) is expected_type for s in obj)
|
||||
|
||||
|
||||
def test_packer_getbuffer():
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue