test_unpack.py: add tests for ExtraData (#708)

fix ##695

.github/workflows/docs.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Setup Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0

.github/workflows/lint.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-slim
     steps:
       - name: Checkout
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: ruff check
         run: |

.github/workflows/test.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Set up Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0

.github/workflows/test_debug.yml

@@ -0,0 +1,62 @@
+name: Run tests with debug Python
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: 3.14.6
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: /opt/python-debug
+          key: python-debug-${{ runner.os }}-${{ env.PYTHON_VERSION }}
+
+      - name: Install build dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential libssl-dev zlib1g-dev \
+            libbz2-dev libreadline-dev libsqlite3-dev curl libncursesw5-dev \
+            xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+      - name: Build Python with Py_DEBUG
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
+          tar -xzf Python-${PYTHON_VERSION}.tgz
+          cd Python-${PYTHON_VERSION}
+
+          ./configure --with-pydebug --prefix=/opt/python-debug
+          make -j4
+          sudo make install
+
+      - name: Create venv from debug Python
+        run: |
+          PYDBG_BIN="/opt/python-debug/bin/python3"
+          "$PYDBG_BIN" -m venv .venv
+          echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
+          echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
+
+      - name: Prepare
+        run: |
+          python -m pip install -r requirements.txt pytest
+
+      - name: Build
+        run: |
+          make cython
+          pip install .
+
+      - name: Test (C extension)
+        run: |
+          pytest -v test
+
+      - name: Test (pure Python fallback)
+        run: |
+          MSGPACK_PUREPYTHON=1 pytest -v test

.github/workflows/wheel.yml

@@ -26,7 +26,7 @@ jobs:
     name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }}
 
     steps:
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: "3.x"
@@ -44,11 +44,11 @@ jobs:
           platforms: ${{ matrix.cibw_archs }}
 
       - name: Build
-        uses: pypa/cibuildwheel@8d2b08b68458a16aeb24b64e68a09ab1c8e82084 # v3.4.1
+        uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
-          CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64"
+          CIBW_SKIP: "cp38-* cp39-* cp310-win_arm64"
           CIBW_ARCHS: ${{ matrix.cibw_archs || 'auto' }}
 
       - name: Build sdist

.gitignore

@@ -15,3 +15,4 @@ msgpack/*.cpp
 /tags
 /docs/_build
 .cache
+uv.lock

CHANGELOG.md

@@ -1,6 +1,15 @@
+# 1.2.1
+
+Release Date: 2026-06-19
+
+Fix a segfault when calling `Unpacker.unpack()` or `Unpacker.skip()` after an unpacking failure.
+But note that reusing the same `Unpacker` instance after an unpacking failure is not supported.
+Please create a new `Unpacker` instance instead. GHSA-6v7p-g79w-8964
+
+
 # 1.2.0
 
-Release Date: TBD
+Release Date: 2026-06-11
 
 - Support free threaded Python. #654, #686
 - Dropped support for Python 3.9. #656

README.md

@@ -72,6 +72,10 @@ for unpacked in unpacker:
     print(unpacked)
 ```
 
+> [!IMPORTANT]
+> If `Unpacker.unpack()` stops with an exception other than `OutOfData`, that `Unpacker` cannot be reused.
+> Create a new `Unpacker` when reading another stream.
+
 
 ### Packing/unpacking of custom data types
 

msgpack/__init__.py

@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 2, 0)
+version = (1, 2, 1)
-__version__ = "1.2.0rc1"
+__version__ = "1.2.1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

msgpack/_unpacker.pyx

@@ -46,11 +46,12 @@ cdef extern from "unpack.h":
         Py_ssize_t count
 
     ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
-                               Py_ssize_t len, Py_ssize_t* off) except? -1
+                               Py_ssize_t len, Py_ssize_t* off) except -1
     execute_fn unpack_construct
     execute_fn unpack_skip
     execute_fn read_array_header
     execute_fn read_map_header
+
     void unpack_init(unpack_context* ctx)
     object unpack_data(unpack_context* ctx)
     void unpack_clear(unpack_context* ctx)
@@ -197,6 +198,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
         if off < buf_len:
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
+
     unpack_clear(&ctx)
     if ret == 0:
         raise ValueError("Unpack failed: incomplete input")
@@ -204,8 +206,6 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
         raise FormatError
     elif ret == -3:
         raise StackError
-    elif PyErr_Occurred():
-        raise
     else:
         raise ValueError("Unpack failed: error = %d" % (ret,))
 
@@ -220,7 +220,7 @@ cdef class Unpacker:
         If specified, unpacker reads serialized data from it and `.feed()` is not usable.
 
     :param int read_size:
-        Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
+        Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*.
 
     :param bool use_list:
         If true, unpack msgpack array to Python list.
@@ -303,6 +303,8 @@ cdef class Unpacker:
     Raises ``OutOfData`` when *packed* is incomplete.
     Raises ``FormatError`` when *packed* is not valid msgpack.
     Raises ``StackError`` when *packed* contains too nested.
+    Raises ``RuntimeError`` when ``feed()`` is called while unpacking
+    is in progress (e.g. from a hook).
     Other exceptions can be raised during unpacking.
     """
     cdef unpack_context ctx
@@ -316,6 +318,7 @@ cdef class Unpacker:
     cdef object unicode_errors
     cdef Py_ssize_t max_buffer_size
     cdef uint64_t stream_offset
+    cdef bint _unpacking
 
     def __dealloc__(self):
         unpack_clear(&self.ctx)
@@ -379,6 +382,7 @@ cdef class Unpacker:
         self.buf_head = 0
         self.buf_tail = 0
         self.stream_offset = 0
+        self._unpacking = False
 
         if unicode_errors is not None:
             self.unicode_errors = unicode_errors
@@ -396,6 +400,11 @@ cdef class Unpacker:
         cdef char* buf
         cdef Py_ssize_t buf_len
 
+        if self._unpacking:
+            raise RuntimeError(
+                "Unpacker.feed() cannot be called while unpacking is in progress"
+            )
+
         if self.file_like is not None:
             raise AssertionError(
                     "unpacker.feed() is not be able to use with `file_like`.")
@@ -463,6 +472,8 @@ cdef class Unpacker:
         cdef object obj
         cdef Py_ssize_t prev_head
 
+        self._unpacking = True
+        try:
             while 1:
                 prev_head = self.buf_head
                 if prev_head < self.buf_tail:
@@ -475,7 +486,7 @@ cdef class Unpacker:
                     obj = unpack_data(&self.ctx)
                     unpack_init(&self.ctx)
                     return obj
-            elif ret == 0:
+                if ret == 0:
                     if self.file_like is not None:
                         self.read_from_file()
                         continue
@@ -483,14 +494,16 @@ cdef class Unpacker:
                         raise StopIteration("No more data to unpack.")
                     else:
                         raise OutOfData("No more data to unpack.")
-            elif ret == -2:
+
+                unpack_clear(&self.ctx)
+                if ret == -2:
                     raise FormatError
                 elif ret == -3:
                     raise StackError
-            elif PyErr_Occurred():
-                raise
                 else:
                     raise ValueError("Unpack failed: error = %d" % (ret,))
+        finally:
+            self._unpacking = False
 
     @cython.critical_section
     def read_bytes(self, Py_ssize_t nbytes):

msgpack/fallback.py

@@ -137,7 +137,7 @@ class Unpacker:
         If specified, unpacker reads serialized data from it and `.feed()` is not usable.
 
     :param int read_size:
-        Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
+        Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*.
 
     :param bool use_list:
         If true, unpack msgpack array to Python list.

msgpack/unpack.h

@@ -40,11 +40,6 @@ struct unpack_context;
 typedef struct unpack_context unpack_context;
 typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off);
 
-static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
-{
-    return NULL;
-}
-
 static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
 {
     PyObject *p = PyLong_FromLong((long)d);
@@ -283,6 +278,7 @@ static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timest
         ts->tv_sec = _msgpack_load64(int64_t, buf + 4);
         return 0;
     default:
+        PyErr_Format(PyExc_ValueError, "invalid timestamp data (length %u)", buflen);
         return -1;
     }
 }
@@ -336,12 +332,18 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos
         else if (u->timestamp == 3) {  // datetime
             // Calculate datetime using epoch + delta
             // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps
+            int64_t days = ts.tv_sec / (24*3600);
+            if (days < INT_MIN || days > INT_MAX) {
+                PyErr_Format(PyExc_OverflowError,
+                            "days=%lld; too large to convert to C int", days);
+                return -1;
+            }
             PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType);
             if (epoch == NULL) {
                 return -1;
             }
 
-            PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000);
+            PyObject* d = PyDelta_FromDSU((int)days, ts.tv_sec%(24*3600), ts.tv_nsec / 1000);
             if (d == NULL) {
                 Py_DECREF(epoch);
                 return -1;

msgpack/unpack_template.h

@@ -35,11 +35,6 @@ struct unpack_context {
     unsigned int cs;
     unsigned int trail;
     unsigned int top;
-    /*
-    unpack_stack* stack;
-    unsigned int stack_size;
-    unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
-    */
     unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
@@ -49,22 +44,9 @@ static inline void unpack_init(unpack_context* ctx)
     ctx->cs = CS_HEADER;
     ctx->trail = 0;
     ctx->top = 0;
-    /*
+    ctx->stack[0].obj = NULL;
-    ctx->stack = ctx->embed_stack;
-    ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
-    */
-    ctx->stack[0].obj = unpack_callback_root(&ctx->user);
 }
 
-/*
-static inline void unpack_destroy(unpack_context* ctx)
-{
-    if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
-        free(ctx->stack);
-    }
-}
-*/
-
 static inline PyObject* unpack_data(unpack_context* ctx)
 {
     return (ctx)->stack[0].obj;
@@ -72,15 +54,14 @@ static inline PyObject* unpack_data(unpack_context* ctx)
 
 static inline void unpack_clear(unpack_context *ctx)
 {
-    unsigned int i;
+    for (unsigned int i = 0; i < ctx->top; i++) {
-    for (i = 1; i < ctx->top; i++) {
-        Py_CLEAR(ctx->stack[i].obj);
         /* map_key holds a live reference only while waiting for the value */
         if (ctx->stack[i].ct == CT_MAP_VALUE) {
             Py_CLEAR(ctx->stack[i].map_key);
         }
+        Py_CLEAR(ctx->stack[i].obj);
     }
-    Py_CLEAR(ctx->stack[0].obj);
+    unpack_init(ctx);
 }
 
 static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
@@ -95,9 +76,6 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char
     unsigned int cs = ctx->cs;
     unsigned int top = ctx->top;
     unpack_stack* stack = ctx->stack;
-    /*
-    unsigned int stack_size = ctx->stack_size;
-    */
     unpack_user* user = &ctx->user;
 
     PyObject* obj = NULL;
@@ -320,6 +298,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char
                 start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
 
             default:
+                PyErr_Format(PyExc_RuntimeError, "Invalid state: %d", cs);
                 goto _failed;
             }
         }
@@ -344,6 +323,7 @@ _push:
         goto _header_again;
     case CT_MAP_VALUE:
         if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+        c->map_key = NULL;
         if(++c->count == c->size) {
             obj = c->obj;
             if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
@@ -355,6 +335,7 @@ _push:
         goto _header_again;
 
     default:
+        PyErr_Format(PyExc_RuntimeError, "Invalid container type: %u", c->ct);
         goto _failed;
     }
 
@@ -406,10 +387,18 @@ _end:
 #undef start_container
 
 static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) {
-    return unpack_execute(1, ctx, data, len, off);
+    int ret = unpack_execute(1, ctx, data, len, off);
+    if (ret == -1) {
+        unpack_clear(ctx);
+    }
+    return ret;
 }
 static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) {
-    return unpack_execute(0, ctx, data, len, off);
+    int ret = unpack_execute(0, ctx, data, len, off);
+    if (ret == -1) {
+        unpack_clear(ctx);
+    }
+    return ret;
 }
 
 #define unpack_container_header read_array_header

requirements.txt

@@ -1,3 +1,4 @@
-Cython==3.2.1
+cython==3.2.5
 setuptools==78.1.1
+pytest
 build

test/test_except.py

@@ -33,6 +33,22 @@ def test_raise_from_object_hook():
         object_pairs_hook=hook,
     )
 
+    up = Unpacker(object_hook=hook)
+
+    def up_unpack(x):
+        up.feed(x)
+        return up.unpack()
+
+    raises(DummyException, up_unpack, packb({}))
+    raises(DummyException, up_unpack, packb({"fizz": "buzz"}))
+    raises(DummyException, up_unpack, packb({"fizz": "buzz"}))
+    raises(DummyException, up_unpack, packb({"fizz": {"buzz": "spam"}}))
+    raises(
+        DummyException,
+        up_unpack,
+        packb({"fizz": {"buzz": "spam"}}),
+    )
+
 
 def test_raise_from_list_hook():
     def hook(lst: list) -> list:
@@ -138,3 +154,13 @@ def test_strict_map_key_with_object_pairs_hook():
     packed = packb(valid, use_bin_type=True)
     result = unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list)
     assert result == [("key", "value")]
+
+
+def test_unpacker_should_not_crash_after_exception():
+    up = Unpacker()  # default: strict_map_key=True
+    up.feed(b"\x83\x73\xc4\x00")  # fixmap(3): int key (rejected) + empty bin8
+    try:
+        up.unpack()  # ValueError: int is not allowed for map key ...
+    except Exception:
+        pass
+    up.skip()  # SIGSEGV (resumes from a corrupt parser context)

test/test_timestamp.py

@@ -176,3 +176,9 @@ def test_pack_datetime_without_tzinfo():
     packed = msgpack.packb(dt, datetime=True)
     unpacked = msgpack.unpackb(packed, timestamp=3)
     assert unpacked == dt
+
+
+def test_too_large_timestamp():
+    # When timestamp64 is too large, conversion to datetime fails due to int64 -> int32 conversion.
+    # https://github.com/msgpack/msgpack-python/issues/696
+    print(msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8, timestamp=3))

test/test_unpack.py

@@ -5,7 +5,15 @@ from io import BytesIO
 
 from pytest import mark, raises
 
-from msgpack import ExtType, OutOfData, Unpacker, packb
+from msgpack import (
+    ExtraData,
+    ExtType,
+    OutOfData,
+    Unpacker,
+    packb,
+    unpack,
+    unpackb,
+)
 
 
 def test_unpack_array_header_from_file():
@@ -123,3 +131,39 @@ def test_unpacker_reinit_clears_partial_state():
 
     unpacker.feed(packb({"a": 1}))
     assert unpacker.unpack() == {"a": 1}
+
+
+@mark.skipif(
+    Unpacker.__module__ == "msgpack.fallback",
+    reason="reentrant guard is implemented in C extension only",
+)
+def test_unpacker_reentrant_feed():
+    import struct
+
+    def ext_hook(code, data):
+        # re-entrant feed on the SAME unpacker, large enough to force a buffer realloc
+        up.feed(b"\xc0" * 100)
+        return 0
+
+    up = Unpacker(ext_hook=ext_hook, max_buffer_size=64 * 1024 * 1024)
+    # array(11): [ ExtType(code=5, data=b'A') (fires the re-entrant hook), then 10 more elements ]
+    up.feed(b"\xdc" + struct.pack(">H", 11) + b"\xd4\x05A" + b"\x2a" * 10)
+    with raises(RuntimeError):
+        up.unpack()
+
+
+def test_unpackb_raises_extra_data_with_trailing_bytes():
+    packed = packb(42) + packb("trailing")
+    with raises(ExtraData) as exc_info:
+        unpackb(packed)
+    err = exc_info.value
+    assert err.unpacked == 42
+    assert err.extra == packb("trailing")
+
+
+def test_unpack_raises_extra_data_on_stream_with_trailing_bytes():
+    stream = BytesIO(packb(100) + packb(200))
+    with raises(ExtraData) as exc_info:
+        unpack(stream)
+    assert exc_info.value.unpacked == 100
+    assert exc_info.value.extra == packb(200)