Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 06:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

Backport fast alternate io.BytesIO implementation.

Browse source 
Merged r62778, r62779, r62802, r62806, r62807, r62808, r62809, r62844,
r62846, r62952, r62956.
This commit is contained in:
Alexandre Vassalotti 2008-05-09 21:49:43 +00:00
parent 81673b7b63
commit 1aed624f7c
10 changed files with 1284 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

89
Lib/io.py
View file

@ -494,6 +494,7 @@ def readline(self, limit = -1):
files, the newlines argument to open can be used to select the line
terminator(s) recognized.
"""
self._checkClosed()
if hasattr(self, "peek"):
def nreadahead():
readahead = self.peek(1)
@ -508,6 +509,8 @@ def nreadahead():
return 1
if limit is None:
limit = -1
if not isinstance(limit, (int, long)):
raise TypeError("limit must be an integer")
res = bytearray()
while limit < 0 or len(res) < limit:
b = self.read(nreadahead())
@ -536,6 +539,10 @@ def readlines(self, hint=None):
lines so far exceeds hint.
"""
if hint is None:
hint = -1
if not isinstance(hint, (int, long)):
raise TypeError("hint must be an integer")
if hint <= 0:
return list(self)
n = 0
lines = []
@ -686,7 +693,7 @@ def readinto(self, b):
import array
if not isinstance(b, array.array):
raise err
b[:n] = array.array('b', data)
b[:n] = array.array(b'b', data)
return n
def write(self, b):
@ -729,6 +736,8 @@ def truncate(self, pos=None):
if pos is None:
pos = self.tell()
# XXX: Should seek() be used, instead of passing the position
# XXX directly to truncate?
return self.raw.truncate(pos)
### Flush and close ###
@ -768,7 +777,7 @@ def isatty(self):
return self.raw.isatty()
class BytesIO(BufferedIOBase):
class _BytesIO(BufferedIOBase):
"""Buffered I/O implementation using an in-memory bytes buffer."""
@ -777,20 +786,28 @@ class BytesIO(BufferedIOBase):
def __init__(self, initial_bytes=None):
buf = bytearray()
if initial_bytes is not None:
buf += initial_bytes
buf += bytearray(initial_bytes)
self._buffer = buf
self._pos = 0
def getvalue(self):
"""Return the bytes value (contents) of the buffer
"""
if self.closed:
raise ValueError("getvalue on closed file")
return bytes(self._buffer)
def read(self, n=None):
if self.closed:
raise ValueError("read from closed file")
if n is None:
n = -1
if not isinstance(n, (int, long)):
raise TypeError("argument must be an integer")
if n < 0:
n = len(self._buffer)
if len(self._buffer) <= self._pos:
return b""
newpos = min(len(self._buffer), self._pos + n)
b = self._buffer[self._pos : newpos]
self._pos = newpos
@ -807,6 +824,8 @@ def write(self, b):
if isinstance(b, unicode):
raise TypeError("can't write unicode to binary stream")
n = len(b)
if n == 0:
return 0
newpos = self._pos + n
if newpos > len(self._buffer):
# Inserts null bytes between the current end of the file
@ -818,28 +837,38 @@ def write(self, b):
return n
def seek(self, pos, whence=0):
if self.closed:
raise ValueError("seek on closed file")
try:
pos = pos.__index__()
except AttributeError as err:
raise TypeError("an integer is required") # from err
if whence == 0:
self._pos = max(0, pos)
if pos < 0:
raise ValueError("negative seek position %r" % (pos,))
self._pos = pos
elif whence == 1:
self._pos = max(0, self._pos + pos)
elif whence == 2:
self._pos = max(0, len(self._buffer) + pos)
else:
raise IOError("invalid whence value")
raise ValueError("invalid whence value")
return self._pos
def tell(self):
if self.closed:
raise ValueError("tell on closed file")
return self._pos
def truncate(self, pos=None):
if self.closed:
raise ValueError("truncate on closed file")
if pos is None:
pos = self._pos
elif pos < 0:
raise ValueError("negative truncate position %r" % (pos,))
del self._buffer[pos:]
return pos
return self.seek(pos)
def readable(self):
return True
@ -850,6 +879,16 @@ def writable(self):
def seekable(self):
return True
# Use the faster implementation of BytesIO if available
try:
import _bytesio
class BytesIO(_bytesio._BytesIO, BufferedIOBase):
__doc__ = _bytesio._BytesIO.__doc__
except ImportError:
BytesIO = _BytesIO
class BufferedReader(_BufferedIOMixin):
@ -983,6 +1022,12 @@ def write(self, b):
raise BlockingIOError(e.errno, e.strerror, overage)
return written
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.raw.tell()
return self.raw.truncate(pos)
def flush(self):
if self.closed:
raise ValueError("flush of closed file")
@ -1102,6 +1147,13 @@ def tell(self):
else:
return self.raw.tell() - len(self._read_buf)
def truncate(self, pos=None):
if pos is None:
pos = self.tell()
# Use seek to flush the read buffer.
self.seek(pos)
return BufferedWriter.truncate(self)
def read(self, n=None):
if n is None:
n = -1
@ -1150,11 +1202,7 @@ def write(self, s):
def truncate(self, pos = None):
"""Truncate size to pos."""
self.flush()
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
self._unsupported("truncate")
def readline(self):
"""Read until newline or EOF.
@ -1351,6 +1399,12 @@ def line_buffering(self):
def seekable(self):
return self._seekable
def readable(self):
return self.buffer.readable()
def writable(self):
return self.buffer.writable()
def flush(self):
self.buffer.flush()
self._telling = self._seekable
@ -1542,7 +1596,16 @@ def tell(self):
finally:
decoder.setstate(saved_state)
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
def seek(self, cookie, whence=0):
if self.closed:
raise ValueError("tell on closed file")
if not self._seekable:
raise IOError("underlying stream is not seekable")
if whence == 1: # seek relative to current position
@ -1629,8 +1692,12 @@ def next(self):
return line
def readline(self, limit=None):
if self.closed:
raise ValueError("read from closed file")
if limit is None:
limit = -1
if not isinstance(limit, (int, long)):
raise TypeError("limit must be an integer")
# Grab all the decoded text (we will rewind any extra bits later).
line = self._get_decoded_chars()

8
Lib/test/test_io.py
View file

@ -97,7 +97,7 @@ def write_ops(self, f):
self.assertEqual(f.seek(-1, 2), 13)
self.assertEqual(f.tell(), 13)
self.assertEqual(f.truncate(12), 12)
self.assertEqual(f.tell(), 13)
self.assertEqual(f.tell(), 12)
self.assertRaises(TypeError, f.seek, 0.0)
def read_ops(self, f, buffered=False):
@ -142,7 +142,7 @@ def large_file_ops(self, f):
self.assertEqual(f.tell(), self.LARGE + 2)
self.assertEqual(f.seek(0, 2), self.LARGE + 2)
self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1)
self.assertEqual(f.tell(), self.LARGE + 2)
self.assertEqual(f.tell(), self.LARGE + 1)
self.assertEqual(f.seek(0, 2), self.LARGE + 1)
self.assertEqual(f.seek(-1, 2), self.LARGE)
self.assertEqual(f.read(2), b"x")
@ -726,6 +726,7 @@ def testNewlinesOutput(self):
txt.write("BB\nCCC\n")
txt.write("X\rY\r\nZ")
txt.flush()
self.assertEquals(buf.closed, False)
self.assertEquals(buf.getvalue(), expected)
def testNewlines(self):
@ -806,7 +807,8 @@ def testNewlinesOutput(self):
txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline)
txt.write(data)
txt.close()
self.assertEquals(buf.getvalue(), expected)
self.assertEquals(buf.closed, True)
self.assertRaises(ValueError, buf.getvalue)
finally:
os.linesep = save_linesep

416
Lib/test/test_memoryio.py Normal file
View file

@ -0,0 +1,416 @@
"""Unit tests for memory-based file-like objects.
StringIO -- for unicode strings
BytesIO -- for bytes
"""
from __future__ import unicode_literals
import unittest
from test import test_support
import io
import sys
import array
try:
import _bytesio
has_c_implementation = True
except ImportError:
has_c_implementation = False
class MemoryTestMixin:
def write_ops(self, f, t):
self.assertEqual(f.write(t("blah.")), 5)
self.assertEqual(f.seek(0), 0)
self.assertEqual(f.write(t("Hello.")), 6)
self.assertEqual(f.tell(), 6)
self.assertEqual(f.seek(5), 5)
self.assertEqual(f.tell(), 5)
self.assertEqual(f.write(t(" world\n\n\n")), 9)
self.assertEqual(f.seek(0), 0)
self.assertEqual(f.write(t("h")), 1)
self.assertEqual(f.truncate(12), 12)
self.assertEqual(f.tell(), 12)
def test_write(self):
buf = self.buftype("hello world\n")
memio = self.ioclass(buf)
self.write_ops(memio, self.buftype)
self.assertEqual(memio.getvalue(), buf)
memio = self.ioclass()
self.write_ops(memio, self.buftype)
self.assertEqual(memio.getvalue(), buf)
self.assertRaises(TypeError, memio.write, None)
memio.close()
self.assertRaises(ValueError, memio.write, self.buftype(""))
def test_writelines(self):
buf = self.buftype("1234567890")
memio = self.ioclass()
self.assertEqual(memio.writelines([buf] * 100), None)
self.assertEqual(memio.getvalue(), buf * 100)
memio.writelines([])
self.assertEqual(memio.getvalue(), buf * 100)
memio = self.ioclass()
self.assertRaises(TypeError, memio.writelines, [buf] + [1])
self.assertEqual(memio.getvalue(), buf)
self.assertRaises(TypeError, memio.writelines, None)
memio.close()
self.assertRaises(ValueError, memio.writelines, [])
def test_writelines_error(self):
memio = self.ioclass()
def error_gen():
yield self.buftype('spam')
raise KeyboardInterrupt
self.assertRaises(KeyboardInterrupt, memio.writelines, error_gen())
def test_truncate(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertRaises(ValueError, memio.truncate, -1)
memio.seek(6)
self.assertEqual(memio.truncate(), 6)
self.assertEqual(memio.getvalue(), buf[:6])
self.assertEqual(memio.truncate(4), 4)
self.assertEqual(memio.getvalue(), buf[:4])
self.assertEqual(memio.tell(), 4)
memio.write(buf)
self.assertEqual(memio.getvalue(), buf[:4] + buf)
pos = memio.tell()
self.assertEqual(memio.truncate(None), pos)
self.assertEqual(memio.tell(), pos)
self.assertRaises(TypeError, memio.truncate, '0')
memio.close()
self.assertRaises(ValueError, memio.truncate, 0)
def test_init(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.getvalue(), buf)
memio = self.ioclass(None)
self.assertEqual(memio.getvalue(), self.EOF)
memio.__init__(buf * 2)
self.assertEqual(memio.getvalue(), buf * 2)
memio.__init__(buf)
self.assertEqual(memio.getvalue(), buf)
def test_read(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.read(0), self.EOF)
self.assertEqual(memio.read(1), buf[:1])
self.assertEqual(memio.read(4), buf[1:5])
self.assertEqual(memio.read(900), buf[5:])
self.assertEqual(memio.read(), self.EOF)
memio.seek(0)
self.assertEqual(memio.read(), buf)
self.assertEqual(memio.read(), self.EOF)
self.assertEqual(memio.tell(), 10)
memio.seek(0)
self.assertEqual(memio.read(-1), buf)
memio.seek(0)
self.assertEqual(type(memio.read()), type(buf))
memio.seek(100)
self.assertEqual(type(memio.read()), type(buf))
memio.seek(0)
self.assertEqual(memio.read(None), buf)
self.assertRaises(TypeError, memio.read, '')
memio.close()
self.assertRaises(ValueError, memio.read)
def test_readline(self):
buf = self.buftype("1234567890\n")
memio = self.ioclass(buf * 2)
self.assertEqual(memio.readline(0), self.EOF)
self.assertEqual(memio.readline(), buf)
self.assertEqual(memio.readline(), buf)
self.assertEqual(memio.readline(), self.EOF)
memio.seek(0)
self.assertEqual(memio.readline(5), buf[:5])
self.assertEqual(memio.readline(5), buf[5:10])
self.assertEqual(memio.readline(5), buf[10:15])
memio.seek(0)
self.assertEqual(memio.readline(-1), buf)
memio.seek(0)
self.assertEqual(memio.readline(0), self.EOF)
buf = self.buftype("1234567890\n")
memio = self.ioclass((buf * 3)[:-1])
self.assertEqual(memio.readline(), buf)
self.assertEqual(memio.readline(), buf)
self.assertEqual(memio.readline(), buf[:-1])
self.assertEqual(memio.readline(), self.EOF)
memio.seek(0)
self.assertEqual(type(memio.readline()), type(buf))
self.assertEqual(memio.readline(None), buf)
self.assertRaises(TypeError, memio.readline, '')
memio.close()
self.assertRaises(ValueError, memio.readline)
def test_readlines(self):
buf = self.buftype("1234567890\n")
memio = self.ioclass(buf * 10)
self.assertEqual(memio.readlines(), [buf] * 10)
memio.seek(5)
self.assertEqual(memio.readlines(), [buf[5:]] + [buf] * 9)
memio.seek(0)
self.assertEqual(memio.readlines(15), [buf] * 2)
memio.seek(0)
self.assertEqual(memio.readlines(-1), [buf] * 10)
memio.seek(0)
self.assertEqual(memio.readlines(0), [buf] * 10)
memio.seek(0)
self.assertEqual(type(memio.readlines()[0]), type(buf))
memio.seek(0)
self.assertEqual(memio.readlines(None), [buf] * 10)
self.assertRaises(TypeError, memio.readlines, '')
memio.close()
self.assertRaises(ValueError, memio.readlines)
def test_iterator(self):
buf = self.buftype("1234567890\n")
memio = self.ioclass(buf * 10)
self.assertEqual(iter(memio), memio)
self.failUnless(hasattr(memio, '__iter__'))
self.failUnless(hasattr(memio, 'next'))
i = 0
for line in memio:
self.assertEqual(line, buf)
i += 1
self.assertEqual(i, 10)
memio.seek(0)
i = 0
for line in memio:
self.assertEqual(line, buf)
i += 1
self.assertEqual(i, 10)
memio = self.ioclass(buf * 2)
memio.close()
self.assertRaises(ValueError, memio.next)
def test_getvalue(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.getvalue(), buf)
memio.read()
self.assertEqual(memio.getvalue(), buf)
self.assertEqual(type(memio.getvalue()), type(buf))
memio = self.ioclass(buf * 1000)
self.assertEqual(memio.getvalue()[-3:], self.buftype("890"))
memio = self.ioclass(buf)
memio.close()
self.assertRaises(ValueError, memio.getvalue)
def test_seek(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
memio.read(5)
self.assertRaises(ValueError, memio.seek, -1)
self.assertRaises(ValueError, memio.seek, 1, -1)
self.assertRaises(ValueError, memio.seek, 1, 3)
self.assertEqual(memio.seek(0), 0)
self.assertEqual(memio.seek(0, 0), 0)
self.assertEqual(memio.read(), buf)
self.assertEqual(memio.seek(3), 3)
self.assertEqual(memio.seek(0, 1), 3)
self.assertEqual(memio.read(), buf[3:])
self.assertEqual(memio.seek(len(buf)), len(buf))
self.assertEqual(memio.read(), self.EOF)
memio.seek(len(buf) + 1)
self.assertEqual(memio.read(), self.EOF)
self.assertEqual(memio.seek(0, 2), len(buf))
self.assertEqual(memio.read(), self.EOF)
memio.close()
self.assertRaises(ValueError, memio.seek, 0)
def test_overseek(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.seek(len(buf) + 1), 11)
self.assertEqual(memio.read(), self.EOF)
self.assertEqual(memio.tell(), 11)
self.assertEqual(memio.getvalue(), buf)
memio.write(self.EOF)
self.assertEqual(memio.getvalue(), buf)
memio.write(buf)
self.assertEqual(memio.getvalue(), buf + self.buftype('\0') + buf)
def test_tell(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.tell(), 0)
memio.seek(5)
self.assertEqual(memio.tell(), 5)
memio.seek(10000)
self.assertEqual(memio.tell(), 10000)
memio.close()
self.assertRaises(ValueError, memio.tell)
def test_flush(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.flush(), None)
def test_flags(self):
memio = self.ioclass()
self.assertEqual(memio.writable(), True)
self.assertEqual(memio.readable(), True)
self.assertEqual(memio.seekable(), True)
self.assertEqual(memio.isatty(), False)
self.assertEqual(memio.closed, False)
memio.close()
self.assertEqual(memio.writable(), True)
self.assertEqual(memio.readable(), True)
self.assertEqual(memio.seekable(), True)
self.assertRaises(ValueError, memio.isatty)
self.assertEqual(memio.closed, True)
def test_subclassing(self):
buf = self.buftype("1234567890")
def test1():
class MemIO(self.ioclass):
pass
m = MemIO(buf)
return m.getvalue()
def test2():
class MemIO(self.ioclass):
def __init__(me, a, b):
self.ioclass.__init__(me, a)
m = MemIO(buf, None)
return m.getvalue()
self.assertEqual(test1(), buf)
self.assertEqual(test2(), buf)
class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
@staticmethod
def buftype(s):
return s.encode("ascii")
ioclass = io._BytesIO
EOF = b""
def test_read1(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertRaises(TypeError, memio.read1)
self.assertEqual(memio.read(), buf)
def test_readinto(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
b = bytearray(b"hello")
self.assertEqual(memio.readinto(b), 5)
self.assertEqual(b, b"12345")
self.assertEqual(memio.readinto(b), 5)
self.assertEqual(b, b"67890")
self.assertEqual(memio.readinto(b), 0)
self.assertEqual(b, b"67890")
b = bytearray(b"hello world")
memio.seek(0)
self.assertEqual(memio.readinto(b), 10)
self.assertEqual(b, b"1234567890d")
b = bytearray(b"")
memio.seek(0)
self.assertEqual(memio.readinto(b), 0)
self.assertEqual(b, b"")
self.assertRaises(TypeError, memio.readinto, '')
a = array.array(b'b', map(ord, b"hello world"))
memio = self.ioclass(buf)
memio.readinto(a)
self.assertEqual(a.tostring(), b"1234567890d")
memio.close()
self.assertRaises(ValueError, memio.readinto, b)
def test_relative_seek(self):
buf = self.buftype("1234567890")
memio = self.ioclass(buf)
self.assertEqual(memio.seek(-1, 1), 0)
self.assertEqual(memio.seek(3, 1), 3)
self.assertEqual(memio.seek(-4, 1), 0)
self.assertEqual(memio.seek(-1, 2), 9)
self.assertEqual(memio.seek(1, 1), 10)
self.assertEqual(memio.seek(1, 2), 11)
memio.seek(-3, 2)
self.assertEqual(memio.read(), buf[-3:])
memio.seek(0)
memio.seek(1, 1)
self.assertEqual(memio.read(), buf[1:])
def test_unicode(self):
memio = self.ioclass()
self.assertRaises(TypeError, self.ioclass, "1234567890")
self.assertRaises(TypeError, memio.write, "1234567890")
self.assertRaises(TypeError, memio.writelines, ["1234567890"])
def test_bytes_array(self):
buf = b"1234567890"
a = array.array(b'b', map(ord, buf))
memio = self.ioclass(a)
self.assertEqual(memio.getvalue(), buf)
self.assertEqual(memio.write(a), 10)
self.assertEqual(memio.getvalue(), buf)
class PyStringIOTest(MemoryTestMixin, unittest.TestCase):
buftype = unicode
ioclass = io.StringIO
EOF = ""
def test_relative_seek(self):
memio = self.ioclass()
self.assertRaises(IOError, memio.seek, -1, 1)
self.assertRaises(IOError, memio.seek, 3, 1)
self.assertRaises(IOError, memio.seek, -3, 1)
self.assertRaises(IOError, memio.seek, -1, 2)
self.assertRaises(IOError, memio.seek, 1, 1)
self.assertRaises(IOError, memio.seek, 1, 2)
# XXX: For the Python version of io.StringIO, this is highly
# dependent on the encoding used for the underlying buffer.
# def test_widechar(self):
# buf = self.buftype("\U0002030a\U00020347")
# memio = self.ioclass(buf)
#
# self.assertEqual(memio.getvalue(), buf)
# self.assertEqual(memio.write(buf), len(buf))
# self.assertEqual(memio.tell(), len(buf))
# self.assertEqual(memio.getvalue(), buf)
# self.assertEqual(memio.write(buf), len(buf))
# self.assertEqual(memio.tell(), len(buf) * 2)
# self.assertEqual(memio.getvalue(), buf + buf)
if has_c_implementation:
class CBytesIOTest(PyBytesIOTest):
ioclass = io.BytesIO
def test_main():
tests = [PyBytesIOTest, PyStringIOTest]
if has_c_implementation:
tests.extend([CBytesIOTest])
test_support.run_unittest(*tests)
if __name__ == '__main__':
test_main()

755
Modules/_bytesio.c Normal file
View file

@ -0,0 +1,755 @@
#include "Python.h"
typedef struct {
PyObject_HEAD
char *buf;
Py_ssize_t pos;
Py_ssize_t string_size;
size_t buf_size;
} BytesIOObject;
#define CHECK_CLOSED(self) \
if ((self)->buf == NULL) { \
PyErr_SetString(PyExc_ValueError, \
"I/O operation on closed file."); \
return NULL; \
}
/* Internal routine to get a line from the buffer of a BytesIO
object. Returns the length between the current position to the
next newline character. */
static Py_ssize_t
get_line(BytesIOObject *self, char **output)
{
char *n;
const char *str_end;
Py_ssize_t len;
assert(self->buf != NULL);
/* Move to the end of the line, up to the end of the string, s. */
str_end = self->buf + self->string_size;
for (n = self->buf + self->pos;
n < str_end && *n != '\n';
n++);
/* Skip the newline character */
if (n < str_end)
n++;
/* Get the length from the current position to the end of the line. */
len = n - (self->buf + self->pos);
*output = self->buf + self->pos;
assert(len >= 0);
assert(self->pos < PY_SSIZE_T_MAX - len);
self->pos += len;
return len;
}
/* Internal routine for changing the size of the buffer of BytesIO objects.
The caller should ensure that the 'size' argument is non-negative. Returns
0 on success, -1 otherwise. */
static int
resize_buffer(BytesIOObject *self, size_t size)
{
/* Here, unsigned types are used to avoid dealing with signed integer
overflow, which is undefined in C. */
size_t alloc = self->buf_size;
char *new_buf = NULL;
assert(self->buf != NULL);
/* For simplicity, stay in the range of the signed type. Anyway, Python
doesn't allow strings to be longer than this. */
if (size > PY_SSIZE_T_MAX)
goto overflow;
if (size < alloc / 2) {
/* Major downsize; resize down to exact size. */
alloc = size + 1;
}
else if (size < alloc) {
/* Within allocated size; quick exit */
return 0;
}
else if (size <= alloc * 1.125) {
/* Moderate upsize; overallocate similar to list_resize() */
alloc = size + (size >> 3) + (size < 9 ? 3 : 6);
}
else {
/* Major upsize; resize up to exact size */
alloc = size + 1;
}
if (alloc > ((size_t)-1) / sizeof(char))
goto overflow;
new_buf = (char *)PyMem_Realloc(self->buf, alloc * sizeof(char));
if (new_buf == NULL) {
PyErr_NoMemory();
return -1;
}
self->buf_size = alloc;
self->buf = new_buf;
return 0;
overflow:
PyErr_SetString(PyExc_OverflowError,
"new buffer size too large");
return -1;
}
/* Internal routine for writing a string of bytes to the buffer of a BytesIO
object. Returns the number of bytes wrote, or -1 on error. */
static Py_ssize_t
write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len)
{
assert(self->buf != NULL);
assert(self->pos >= 0);
assert(len >= 0);
if ((size_t)self->pos + len > self->buf_size) {
if (resize_buffer(self, (size_t)self->pos + len) < 0)
return -1;
}
if (self->pos > self->string_size) {
/* In case of overseek, pad with null bytes the buffer region between
the end of stream and the current position.
0 lo string_size hi
| |<---used--->|<----------available----------->|
| | <--to pad-->|<---to write---> |
0 buf position
*/
memset(self->buf + self->string_size, '\0',
(self->pos - self->string_size) * sizeof(char));
}
/* Copy the data to the internal buffer, overwriting some of the existing
data if self->pos < self->string_size. */
memcpy(self->buf + self->pos, bytes, len);
self->pos += len;
/* Set the new length of the internal string if it has changed. */
if (self->string_size < self->pos) {
self->string_size = self->pos;
}
return len;
}
static PyObject *
bytesio_get_closed(BytesIOObject *self)
{
if (self->buf == NULL)
Py_RETURN_TRUE;
else
Py_RETURN_FALSE;
}
/* Generic getter for the writable, readable and seekable properties */
static PyObject *
return_true(BytesIOObject *self)
{
Py_RETURN_TRUE;
}
PyDoc_STRVAR(flush_doc,
"flush() -> None. Does nothing.");
static PyObject *
bytesio_flush(BytesIOObject *self)
{
Py_RETURN_NONE;
}
PyDoc_STRVAR(getval_doc,
"getvalue() -> bytes.\n"
"\n"
"Retrieve the entire contents of the BytesIO object.");
static PyObject *
bytesio_getvalue(BytesIOObject *self)
{
CHECK_CLOSED(self);
return PyString_FromStringAndSize(self->buf, self->string_size);
}
PyDoc_STRVAR(isatty_doc,
"isatty() -> False.\n"
"\n"
"Always returns False since BytesIO objects are not connected\n"
"to a tty-like device.");
static PyObject *
bytesio_isatty(BytesIOObject *self)
{
CHECK_CLOSED(self);
Py_RETURN_FALSE;
}
PyDoc_STRVAR(tell_doc,
"tell() -> current file position, an integer\n");
static PyObject *
bytesio_tell(BytesIOObject *self)
{
CHECK_CLOSED(self);
return PyInt_FromSsize_t(self->pos);
}
PyDoc_STRVAR(read_doc,
"read([size]) -> read at most size bytes, returned as a string.\n"
"\n"
"If the size argument is negative, read until EOF is reached.\n"
"Return an empty string at EOF.");
static PyObject *
bytesio_read(BytesIOObject *self, PyObject *args)
{
Py_ssize_t size, n;
char *output;
PyObject *arg = Py_None;
CHECK_CLOSED(self);
if (!PyArg_ParseTuple(args, "|O:read", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
}
else if (arg == Py_None) {
/* Read until EOF is reached, by default. */
size = -1;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
/* adjust invalid sizes */
n = self->string_size - self->pos;
if (size < 0 || size > n) {
size = n;
if (size < 0)
size = 0;
}
assert(self->buf != NULL);
output = self->buf + self->pos;
self->pos += size;
return PyString_FromStringAndSize(output, size);
}
PyDoc_STRVAR(read1_doc,
"read1(size) -> read at most size bytes, returned as a string.\n"
"\n"
"If the size argument is negative or omitted, read until EOF is reached.\n"
"Return an empty string at EOF.");
static PyObject *
bytesio_read1(BytesIOObject *self, PyObject *n)
{
PyObject *arg, *res;
arg = PyTuple_Pack(1, n);
if (arg == NULL)
return NULL;
res = bytesio_read(self, arg);
Py_DECREF(arg);
return res;
}
PyDoc_STRVAR(readline_doc,
"readline([size]) -> next line from the file, as a string.\n"
"\n"
"Retain newline. A non-negative size argument limits the maximum\n"
"number of bytes to return (an incomplete line may be returned then).\n"
"Return an empty string at EOF.\n");
static PyObject *
bytesio_readline(BytesIOObject *self, PyObject *args)
{
Py_ssize_t size, n;
char *output;
PyObject *arg = Py_None;
CHECK_CLOSED(self);
if (!PyArg_ParseTuple(args, "|O:readline", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
}
else if (arg == Py_None) {
/* No size limit, by default. */
size = -1;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
n = get_line(self, &output);
if (size >= 0 && size < n) {
size = n - size;
n -= size;
self->pos -= size;
}
return PyString_FromStringAndSize(output, n);
}
PyDoc_STRVAR(readlines_doc,
"readlines([size]) -> list of strings, each a line from the file.\n"
"\n"
"Call readline() repeatedly and return a list of the lines so read.\n"
"The optional size argument, if given, is an approximate bound on the\n"
"total number of bytes in the lines returned.\n");
static PyObject *
bytesio_readlines(BytesIOObject *self, PyObject *args)
{
Py_ssize_t maxsize, size, n;
PyObject *result, *line;
char *output;
PyObject *arg = Py_None;
CHECK_CLOSED(self);
if (!PyArg_ParseTuple(args, "|O:readlines", &arg))
return NULL;
if (PyInt_Check(arg)) {
maxsize = PyInt_AsSsize_t(arg);
}
else if (arg == Py_None) {
/* No size limit, by default. */
maxsize = -1;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
size = 0;
result = PyList_New(0);
if (!result)
return NULL;
while ((n = get_line(self, &output)) != 0) {
line = PyString_FromStringAndSize(output, n);
if (!line)
goto on_error;
if (PyList_Append(result, line) == -1) {
Py_DECREF(line);
goto on_error;
}
Py_DECREF(line);
size += n;
if (maxsize > 0 && size >= maxsize)
break;
}
return result;
on_error:
Py_DECREF(result);
return NULL;
}
PyDoc_STRVAR(readinto_doc,
"readinto(bytearray) -> int. Read up to len(b) bytes into b.\n"
"\n"
"Returns number of bytes read (0 for EOF), or None if the object\n"
"is set not to block as has no data to read.");
static PyObject *
bytesio_readinto(BytesIOObject *self, PyObject *buffer)
{
void *raw_buffer;
Py_ssize_t len;
CHECK_CLOSED(self);
if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1)
return NULL;
if (self->pos + len > self->string_size)
len = self->string_size - self->pos;
memcpy(raw_buffer, self->buf + self->pos, len);
assert(self->pos + len < PY_SSIZE_T_MAX);
assert(len >= 0);
self->pos += len;
return PyInt_FromSsize_t(len);
}
PyDoc_STRVAR(truncate_doc,
"truncate([size]) -> int. Truncate the file to at most size bytes.\n"
"\n"
"Size defaults to the current file position, as returned by tell().\n"
"Returns the new size. Imply an absolute seek to the position size.");
static PyObject *
bytesio_truncate(BytesIOObject *self, PyObject *args)
{
Py_ssize_t size;
PyObject *arg = Py_None;
CHECK_CLOSED(self);
if (!PyArg_ParseTuple(args, "|O:truncate", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
}
else if (arg == Py_None) {
/* Truncate to current position if no argument is passed. */
size = self->pos;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
if (size < 0) {
PyErr_Format(PyExc_ValueError,
"negative size value %zd", size);
return NULL;
}
if (size < self->string_size) {
self->string_size = size;
if (resize_buffer(self, size) < 0)
return NULL;
}
self->pos = size;
return PyInt_FromSsize_t(size);
}
static PyObject *
bytesio_iternext(BytesIOObject *self)
{
char *next;
Py_ssize_t n;
CHECK_CLOSED(self);
n = get_line(self, &next);
if (!next || n == 0)
return NULL;
return PyString_FromStringAndSize(next, n);
}
PyDoc_STRVAR(seek_doc,
"seek(pos, whence=0) -> int. Change stream position.\n"
"\n"
"Seek to byte offset pos relative to position indicated by whence:\n"
" 0 Start of stream (the default). pos should be >= 0;\n"
" 1 Current position - pos may be negative;\n"
" 2 End of stream - pos usually negative.\n"
"Returns the new absolute position.");
static PyObject *
bytesio_seek(BytesIOObject *self, PyObject *args)
{
PyObject *pos_obj, *mode_obj;
Py_ssize_t pos;
int mode = 0;
CHECK_CLOSED(self);
/* Special-case for 2.x to prevent floats from passing through.
This only needed to make a test in test_io succeed. */
if (!PyArg_UnpackTuple(args, "seek", 1, 2, &pos_obj, &mode_obj))
return NULL;
if (PyFloat_Check(pos_obj)) {
PyErr_SetString(PyExc_TypeError,
"position argument must be an integer");
return NULL;
}
if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode))
return NULL;
if (pos < 0 && mode == 0) {
PyErr_Format(PyExc_ValueError,
"negative seek value %zd", pos);
return NULL;
}
/* mode 0: offset relative to beginning of the string.
mode 1: offset relative to current position.
mode 2: offset relative the end of the string. */
if (mode == 1) {
if (pos > PY_SSIZE_T_MAX - self->pos) {
PyErr_SetString(PyExc_OverflowError,
"new position too large");
return NULL;
}
pos += self->pos;
}
else if (mode == 2) {
if (pos > PY_SSIZE_T_MAX - self->string_size) {
PyErr_SetString(PyExc_OverflowError,
"new position too large");
return NULL;
}
pos += self->string_size;
}
else if (mode != 0) {
PyErr_Format(PyExc_ValueError,
"invalid whence (%i, should be 0, 1 or 2)", mode);
return NULL;
}
if (pos < 0)
pos = 0;
self->pos = pos;
return PyInt_FromSsize_t(self->pos);
}
PyDoc_STRVAR(write_doc,
"write(bytes) -> int. Write bytes to file.\n"
"\n"
"Return the number of bytes written.");
static PyObject *
bytesio_write(BytesIOObject *self, PyObject *obj)
{
const char *bytes;
Py_ssize_t size;
Py_ssize_t n = 0;
CHECK_CLOSED(self);
/* Special-case in 2.x to prevent unicode objects to pass through. */
if (PyUnicode_Check(obj)) {
PyErr_SetString(PyExc_TypeError,
"expecting a bytes object, got unicode");
return NULL;
}
if (PyObject_AsReadBuffer(obj, (void *)&bytes, &size) < 0)
return NULL;
if (size != 0) {
n = write_bytes(self, bytes, size);
if (n < 0)
return NULL;
}
return PyInt_FromSsize_t(n);
}
PyDoc_STRVAR(writelines_doc,
"writelines(sequence_of_strings) -> None. Write strings to the file.\n"
"\n"
"Note that newlines are not added. The sequence can be any iterable\n"
"object producing strings. This is equivalent to calling write() for\n"
"each string.");
static PyObject *
bytesio_writelines(BytesIOObject *self, PyObject *v)
{
PyObject *it, *item;
PyObject *ret;
CHECK_CLOSED(self);
it = PyObject_GetIter(v);
if (it == NULL)
return NULL;
while ((item = PyIter_Next(it)) != NULL) {
ret = bytesio_write(self, item);
Py_DECREF(item);
if (ret == NULL) {
Py_DECREF(it);
return NULL;
}
Py_DECREF(ret);
}
Py_DECREF(it);
/* See if PyIter_Next failed */
if (PyErr_Occurred())
return NULL;
Py_RETURN_NONE;
}
PyDoc_STRVAR(close_doc,
"close() -> None. Disable all I/O operations.");
static PyObject *
bytesio_close(BytesIOObject *self)
{
if (self->buf != NULL) {
PyMem_Free(self->buf);
self->buf = NULL;
}
Py_RETURN_NONE;
}
static void
bytesio_dealloc(BytesIOObject *self)
{
if (self->buf != NULL) {
PyMem_Free(self->buf);
self->buf = NULL;
}
Py_TYPE(self)->tp_free(self);
}
static PyObject *
bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
BytesIOObject *self;
assert(type != NULL && type->tp_alloc != NULL);
self = (BytesIOObject *)type->tp_alloc(type, 0);
if (self == NULL)
return NULL;
self->string_size = 0;
self->pos = 0;
self->buf_size = 0;
self->buf = (char *)PyMem_Malloc(0);
if (self->buf == NULL) {
Py_DECREF(self);
return PyErr_NoMemory();
}
return (PyObject *)self;
}
static int
bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds)
{
PyObject *initvalue = NULL;
if (!PyArg_ParseTuple(args, "|O:BytesIO", &initvalue))
return -1;
/* In case, __init__ is called multiple times. */
self->string_size = 0;
self->pos = 0;
if (initvalue && initvalue != Py_None) {
PyObject *res;
res = bytesio_write(self, initvalue);
if (res == NULL)
return -1;
Py_DECREF(res);
self->pos = 0;
}
return 0;
}
static PyGetSetDef bytesio_getsetlist[] = {
{"closed", (getter)bytesio_get_closed, NULL,
"True if the file is closed."},
{0}, /* sentinel */
};
static struct PyMethodDef bytesio_methods[] = {
{"readable", (PyCFunction)return_true, METH_NOARGS, NULL},
{"seekable", (PyCFunction)return_true, METH_NOARGS, NULL},
{"writable", (PyCFunction)return_true, METH_NOARGS, NULL},
{"close", (PyCFunction)bytesio_close, METH_NOARGS, close_doc},
{"flush", (PyCFunction)bytesio_flush, METH_NOARGS, flush_doc},
{"isatty", (PyCFunction)bytesio_isatty, METH_NOARGS, isatty_doc},
{"tell", (PyCFunction)bytesio_tell, METH_NOARGS, tell_doc},
{"write", (PyCFunction)bytesio_write, METH_O, write_doc},
{"writelines", (PyCFunction)bytesio_writelines, METH_O, writelines_doc},
{"read1", (PyCFunction)bytesio_read1, METH_O, read1_doc},
{"readinto", (PyCFunction)bytesio_readinto, METH_O, readinto_doc},
{"readline", (PyCFunction)bytesio_readline, METH_VARARGS, readline_doc},
{"readlines", (PyCFunction)bytesio_readlines, METH_VARARGS, readlines_doc},
{"read", (PyCFunction)bytesio_read, METH_VARARGS, read_doc},
{"getvalue", (PyCFunction)bytesio_getvalue, METH_VARARGS, getval_doc},
{"seek", (PyCFunction)bytesio_seek, METH_VARARGS, seek_doc},
{"truncate", (PyCFunction)bytesio_truncate, METH_VARARGS, truncate_doc},
{NULL, NULL} /* sentinel */
};
PyDoc_STRVAR(bytesio_doc,
"BytesIO([buffer]) -> object\n"
"\n"
"Create a buffered I/O implementation using an in-memory bytes\n"
"buffer, ready for reading and writing.");
static PyTypeObject BytesIO_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_bytesio._BytesIO", /*tp_name*/
sizeof(BytesIOObject), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)bytesio_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
bytesio_doc, /*tp_doc*/
0, /*tp_traverse*/
0, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
PyObject_SelfIter, /*tp_iter*/
(iternextfunc)bytesio_iternext, /*tp_iternext*/
bytesio_methods, /*tp_methods*/
0, /*tp_members*/
bytesio_getsetlist, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
(initproc)bytesio_init, /*tp_init*/
0, /*tp_alloc*/
bytesio_new, /*tp_new*/
};
PyMODINIT_FUNC
init_bytesio(void)
{
PyObject *m;
if (PyType_Ready(&BytesIO_Type) < 0)
return;
m = Py_InitModule("_bytesio", NULL);
if (m == NULL)
return;
Py_INCREF(&BytesIO_Type);
PyModule_AddObject(m, "_BytesIO", (PyObject *)&BytesIO_Type);
}

62
Modules/_fileio.c
View file

@ -552,11 +552,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
PyErr_SetString(PyExc_TypeError, "an integer is required");
return NULL;
}
#if !defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj);
#if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLongLong(posobj);
#else
pos = PyLong_Check(posobj) ?
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
pos = PyLong_AsLong(posobj);
#endif
if (PyErr_Occurred())
return NULL;
@ -572,10 +571,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
if (res < 0)
return PyErr_SetFromErrno(PyExc_IOError);
#if !defined(HAVE_LARGEFILE_SUPPORT)
return PyLong_FromLong(res);
#else
#if defined(HAVE_LARGEFILE_SUPPORT)
return PyLong_FromLongLong(res);
#else
return PyLong_FromLong(res);
#endif
}
@ -622,48 +621,29 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
return NULL;
if (posobj == Py_None || posobj == NULL) {
/* Get the current position. */
posobj = portable_lseek(fd, NULL, 1);
if (posobj == NULL)
return NULL;
}
else {
Py_INCREF(posobj);
/* Move to the position to be truncated. */
posobj = portable_lseek(fd, posobj, 0);
}
#if !defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj);
#if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLongLong(posobj);
#else
pos = PyLong_Check(posobj) ?
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
pos = PyLong_AsLong(posobj);
#endif
if (PyErr_Occurred()) {
Py_DECREF(posobj);
if (PyErr_Occurred())
return NULL;
}
#ifdef MS_WINDOWS
/* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
so don't even try using it. */
{
HANDLE hFile;
PyObject *pos2, *oldposobj;
/* store the current position */
oldposobj = portable_lseek(self->fd, NULL, 1);
if (oldposobj == NULL) {
Py_DECREF(posobj);
return NULL;
}
/* Have to move current pos to desired endpoint on Windows. */
errno = 0;
pos2 = portable_lseek(fd, posobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
Py_DECREF(pos2);
/* Truncate. Note that this may grow the file! */
Py_BEGIN_ALLOW_THREADS
@ -676,18 +656,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
errno = EACCES;
}
Py_END_ALLOW_THREADS
if (ret == 0) {
/* Move to the previous position in the file */
pos2 = portable_lseek(fd, oldposobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
}
Py_DECREF(pos2);
Py_DECREF(oldposobj);
}
#else
Py_BEGIN_ALLOW_THREADS
@ -697,7 +665,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
#endif /* !MS_WINDOWS */
if (ret != 0) {
Py_DECREF(posobj);
PyErr_SetFromErrno(PyExc_IOError);
return NULL;
}
@ -791,7 +758,8 @@ PyDoc_STRVAR(seek_doc,
PyDoc_STRVAR(truncate_doc,
"truncate([size: int]) -> None. Truncate the file to at most size bytes.\n"
"\n"
"Size defaults to the current file position, as returned by tell().");
"Size defaults to the current file position, as returned by tell()."
"The current file position is changed to the value of size.");
#endif
PyDoc_STRVAR(tell_doc,

4
Objects/bytesobject.c
View file

@ -315,8 +315,8 @@ bytes_iconcat(PyBytesObject *self, PyObject *other)
Py_buffer vo;
if (_getbuffer(other, &vo) < 0) {
PyErr_Format(PyExc_TypeError, "can't concat bytes to %.100s",
Py_TYPE(self)->tp_name);
PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s",
Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name);
return NULL;
}

4
PC/VS8.0/pythoncore.vcproj
View file

@ -990,6 +990,10 @@
RelativePath="..\..\Modules\_fileio.c"
>
</File>
<File
RelativePath="..\..\Modules\_bytesio.c"
>
</File>
<File
RelativePath="..\..\Modules\_functoolsmodule.c"
>

2
PC/config.c
View file

@ -53,6 +53,7 @@ extern void init_winreg(void);
extern void init_struct(void);
extern void initdatetime(void);
extern void init_fileio(void);
extern void init_bytesio(void);
extern void init_functools(void);
extern void init_json(void);
extern void initzlib(void);
@ -132,6 +133,7 @@ struct _inittab _PyImport_Inittab[] = {
{"_struct", init_struct},
{"datetime", initdatetime},
{"_fileio", init_fileio},
{"_bytesio", init_bytesio},
{"_functools", init_functools},
{"_json", init_json},

4
PCbuild/pythoncore.vcproj
View file

@ -994,6 +994,10 @@
RelativePath="..\Modules\_fileio.c"
>
</File>
<File
RelativePath="..\Modules\_bytesio.c"
>
</File>
<File
RelativePath="..\Modules\_functoolsmodule.c"
>

2
setup.py
View file

@ -436,6 +436,8 @@ def detect_modules(self):
exts.append( Extension('operator', ['operator.c']) )
# Python 3.0 _fileio module
exts.append( Extension("_fileio", ["_fileio.c"]) )
# Python 3.0 _bytesio module
exts.append( Extension("_bytesio", ["_bytesio.c"]) )
# _functools
exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
# _json speedups