Merge pull request #50 from msgpack/skip-reserved

Unpacker skip reserved bytes

msgpack/fallback.py

@@ -57,6 +57,7 @@ TYPE_IMMEDIATE          = 0
 TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
+TYPE_RESERVED           = 4
 
 DEFAULT_RECURSE_LIMIT=511
 
@@ -267,8 +268,10 @@ class Unpacker(object):
             write_bytes(ret)
         return ret
 
-    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+    def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None):
         typ = TYPE_IMMEDIATE
+        n = 0
+        obj = None
         c = self._fb_read(1, write_bytes)
         b = ord(c)
         if   b & 0b10000000 == 0:
@@ -332,7 +335,14 @@ class Unpacker(object):
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             typ = TYPE_MAP
         else:
-            raise UnpackValueError("Unknown header: 0x%x" % b)
+            typ = TYPE_RESERVED
+        return typ, n, obj
+
+    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+        typ = TYPE_RESERVED
+        while typ == TYPE_RESERVED:
+            typ, n, obj = self._read_header(self, execute, write_bytes)
+
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
                 raise UnpackValueError("Expected array")

msgpack/unpack_template.h

@@ -203,18 +203,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				switch(*p) {
 				case 0xc0:  // nil
 					push_simple_value(_nil);
-				//case 0xc1:  // string
+					// reserved.
+				case 0xc1:  // string
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
 					//	again_terminal_trail(NEXT_CS(p), p+1);
 				case 0xc2:  // false
 					push_simple_value(_false);
 				case 0xc3:  // true
 					push_simple_value(_true);
-				//case 0xc4:
+					// reserved.
-				//case 0xc5:
+				case 0xc4:
-				//case 0xc6:
+				case 0xc5:
-				//case 0xc7:
+				case 0xc6:
-				//case 0xc8:
+				case 0xc7:
-				//case 0xc9:
+				case 0xc8:
+				case 0xc9:
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
 				case 0xca:  // float
 				case 0xcb:  // double
 				case 0xcc:  // unsigned int  8
@@ -226,12 +232,15 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xd2:  // signed int 32
 				case 0xd3:  // signed int 64
 					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-				//case 0xd4:
+					// reserved.
-				//case 0xd5:
+				case 0xd4:
-				//case 0xd6:  // big integer 16
+				case 0xd5:
-				//case 0xd7:  // big integer 32
+				case 0xd6:  // big integer 16
-				//case 0xd8:  // big float 16
+				case 0xd7:  // big integer 32
-				//case 0xd9:  // big float 32
+				case 0xd8:  // big float 16
+				case 0xd9:  // big float 32
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
 				case 0xda:  // raw 16
 				case 0xdb:  // raw 32
 				case 0xdc:  // array 16
@@ -240,6 +249,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xdf:  // map 32
 					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
 				default:
+					fprintf(stderr, "failed %x\n", (int)*p);
 					goto _failed;
 				}
 			SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
@@ -250,6 +260,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
 
 			SWITCH_RANGE_DEFAULT
+				fprintf(stderr, "failed2 %x\n", (int)*p);
 			goto _failed;
 			SWITCH_RANGE_END
 				// end CS_HEADER
@@ -422,7 +433,10 @@ _end:
 template <unsigned int fixed_offset, unsigned int var_offset>
 msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
-	assert(len >= *off);
+	for (;;) {
+		if (len < *off) {
+			return 0;
+		}
 		uint32_t size;
 		const unsigned char *const p = (unsigned char*)data + *off;
 
@@ -463,6 +477,22 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 			++*off;
 			size = ((unsigned int)*p) & 0x0f;
 			break;
+			// reserved:
+		case 0xc1:
+		case 0xc4:
+		case 0xc5:
+		case 0xc6:
+		case 0xc7:
+		case 0xc8:
+		case 0xc9:
+		case 0xd4:
+		case 0xd5:
+		case 0xd6:  // big integer 16
+		case 0xd7:  // big integer 32
+		case 0xd8:  // big float 16
+		case 0xd9:  // big float 32
+			++*off;
+			continue;
 		default:
 			PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
 			return -1;
@@ -470,6 +500,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 		msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
 		return 1;
 	}
+}
 
 #undef SWITCH_RANGE_BEGIN
 #undef SWITCH_RANGE

test/test_reserved.py

@@ -0,0 +1,22 @@
+import msgpack
+
+reserved_bytes = [
+        b"\xc1",
+        b"\xc4",
+        b"\xc5",
+        b"\xc6",
+        b"\xc7",
+        b"\xc8",
+        b"\xc9",
+        b"\xd4",
+        b"\xd5",
+        b"\xd6",
+        b"\xd7",
+        b"\xd8",
+        b"\xd9",
+        ]
+
+def test_skip_reserved():
+    packed_list = msgpack.packb([])
+    for b in reserved_bytes:
+        assert msgpack.unpackb(b+packed_list, use_list=1) == []