Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-03 23:21:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

Issue #23215: Multibyte codecs with custom error handlers that ignores errors

Browse source 
consumed too much memory and raised SystemError or MemoryError.
Original patch by Aleksi Torhamo.
This commit is contained in:
Serhiy Storchaka 2015-02-21 01:19:58 +02:00
parent a3712a9a6c
commit a1543cdcd6
3 changed files with 22 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Lib/test/test_multibytecodec.py
View file

@ -44,6 +44,13 @@ def test_errorcallback_longindex(self):
self.assertRaises(IndexError, dec, self.assertRaises(IndexError, dec,
b'apple\x92ham\x93spam', 'test.cjktest') b'apple\x92ham\x93spam', 'test.cjktest')
def test_errorcallback_custom_ignore(self):
# Issue #23215: MemoryError with custom error handlers and multibyte codecs
data = 100 * "\udc00"
codecs.register_error("test.ignore", codecs.ignore_errors)
for enc in ALL_CJKENCODINGS:
self.assertEqual(data.encode(enc, "test.ignore"), b'')
def test_codingspec(self): def test_codingspec(self):
try: try:
for enc in ALL_CJKENCODINGS: for enc in ALL_CJKENCODINGS:

4
Misc/NEWS
View file

@ -13,6 +13,10 @@ Core and Builtins
Library Library
------- -------
- Issue #23215: Multibyte codecs with custom error handlers that ignores errors
consumed too much memory and raised SystemError or MemoryError.
Original patch by Aleksi Torhamo.
- Issue #5700: io.FileIO() called flush() after closing the file. - Issue #5700: io.FileIO() called flush() after closing the file.
flush() was not called in close() if closefd=False. flush() was not called in close() if closefd=False.

19
Modules/cjkcodecs/multibytecodec.c
View file

@ -182,8 +182,10 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
orgsize = PyBytes_GET_SIZE(buf->outobj); orgsize = PyBytes_GET_SIZE(buf->outobj);
incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
if (orgsize > PY_SSIZE_T_MAX - incsize) if (orgsize > PY_SSIZE_T_MAX - incsize) {
PyErr_NoMemory();
return -1; return -1;
}
if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1) if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
return -1; return -1;
@ -194,11 +196,11 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
return 0; return 0;
} }
#define REQUIRE_ENCODEBUFFER(buf, s) { \ #define REQUIRE_ENCODEBUFFER(buf, s) do { \
if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
if (expand_encodebuffer(buf, s) == -1) \ if (expand_encodebuffer(buf, s) == -1) \
goto errorexit; \ goto errorexit; \
} } while(0)
/** /**
@ -332,10 +334,11 @@ multibytecodec_encerror(MultibyteCodec *codec,
assert(PyBytes_Check(retstr)); assert(PyBytes_Check(retstr));
retstrsize = PyBytes_GET_SIZE(retstr); retstrsize = PyBytes_GET_SIZE(retstr);
REQUIRE_ENCODEBUFFER(buf, retstrsize); if (retstrsize > 0) {
REQUIRE_ENCODEBUFFER(buf, retstrsize);
memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize); memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
buf->outbuf += retstrsize; buf->outbuf += retstrsize;
}
newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
if (newpos < 0 && !PyErr_Occurred()) if (newpos < 0 && !PyErr_Occurred())