From 09e5cf28aef05ad07bf885c1b732eb567470199a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:09:18 +0200 Subject: [PATCH 01/20] Issue #22117: Use the _PyTime_t API in _datetime.datetime() constructor * Remove _PyTime_gettimeofday() * Add _PyTime_GetSystemClock() --- Include/pytime.h | 20 ++----- Modules/_datetimemodule.c | 18 ++++-- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 119 +++++--------------------------------- 4 files changed, 35 insertions(+), 124 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 0ff009a9505..919ba30e5e5 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,15 +13,6 @@ functions and constants extern "C" { #endif -#ifdef HAVE_GETTIMEOFDAY -typedef struct timeval _PyTime_timeval; -#else -typedef struct { - time_t tv_sec; /* seconds since Jan. 1, 1970 */ - long tv_usec; /* and microseconds */ -} _PyTime_timeval; -#endif - /* Structure used by time.get_clock_info() */ typedef struct { const char *implementation; @@ -30,11 +21,6 @@ typedef struct { double resolution; } _Py_clock_info_t; -/* Similar to POSIX gettimeofday but cannot fail. If system gettimeofday - * fails or is not available, fall back to lower resolution clocks. - */ -PyAPI_FUNC(void) _PyTime_gettimeofday(_PyTime_timeval *tp); - typedef enum { /* Round towards zero. */ _PyTime_ROUND_DOWN=0, @@ -133,6 +119,12 @@ PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); #endif +/* Get the current time from the system clock. + + The function cannot fail. _PyTime_Init() ensures that the system clock + works. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); + /* Get the current time from the system clock. * Fill clock information if info is not NULL. * Raise an exception and return -1 on error, return 0 on success. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 09285d919de..c3e54f7af43 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7,6 +7,10 @@ #include +#ifdef MS_WINDOWS +# include /* struct timeval */ +#endif + /* Differentiate between building the core module and building extension * modules. */ @@ -4093,6 +4097,8 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_DOWN) == -1) return NULL; + assert(0 <= us && us <= 999999); + return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); } @@ -4103,10 +4109,14 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_timeval t; - _PyTime_gettimeofday(&t); - return datetime_from_timet_and_us(cls, f, t.tv_sec, (int)t.tv_usec, - tzinfo); + _PyTime_t ts = _PyTime_GetSystemClock(); + struct timeval tv; + + if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_FLOOR) < 0) + return NULL; + assert(0 <= tv.tv_usec && tv.tv_usec <= 999999); + + return datetime_from_timet_and_us(cls, f, tv.tv_sec, tv.tv_usec, tzinfo); } /*[clinic input] diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5c54ad6785e..9abb7ccd195 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -15,7 +15,7 @@ #include #ifdef MS_WINDOWS -# include +# include /* struct timeval */ #endif #ifdef WITH_THREAD diff --git a/Python/pytime.c b/Python/pytime.c index d23d9d36b9e..11e3a627ed9 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -19,106 +19,6 @@ #define MS_TO_NS (MS_TO_US * US_TO_NS) #define SEC_TO_NS (SEC_TO_MS * MS_TO_NS) -static int -pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info, int raise) -{ -#ifdef MS_WINDOWS - FILETIME system_time; - ULARGE_INTEGER large; - ULONGLONG microseconds; - - assert(info == NULL || raise); - - GetSystemTimeAsFileTime(&system_time); - large.u.LowPart = system_time.dwLowDateTime; - large.u.HighPart = system_time.dwHighDateTime; - /* 11,644,473,600,000,000: number of microseconds between - the 1st january 1601 and the 1st january 1970 (369 years + 89 leap - days). */ - microseconds = large.QuadPart / 10 - 11644473600000000; - tp->tv_sec = microseconds / SEC_TO_US; - tp->tv_usec = microseconds % SEC_TO_US; - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - - info->implementation = "GetSystemTimeAsFileTime()"; - info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 1; - } - -#else /* MS_WINDOWS */ - int err; -#ifdef HAVE_CLOCK_GETTIME - struct timespec ts; -#endif - - assert(info == NULL || raise); - -#ifdef HAVE_CLOCK_GETTIME - err = clock_gettime(CLOCK_REALTIME, &ts); - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - tp->tv_sec = ts.tv_sec; - tp->tv_usec = ts.tv_nsec / US_TO_NS; - - if (info) { - struct timespec res; - info->implementation = "clock_gettime(CLOCK_REALTIME)"; - info->monotonic = 0; - info->adjustable = 1; - if (clock_getres(CLOCK_REALTIME, &res) == 0) - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - else - info->resolution = 1e-9; - } -#else /* HAVE_CLOCK_GETTIME */ - - /* test gettimeofday() */ -#ifdef GETTIMEOFDAY_NO_TZ - err = gettimeofday(tp); -#else - err = gettimeofday(tp, (struct timezone *)NULL); -#endif - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - - if (info) { - info->implementation = "gettimeofday()"; - info->resolution = 1e-6; - info->monotonic = 0; - info->adjustable = 1; - } -#endif /* !HAVE_CLOCK_GETTIME */ -#endif /* !MS_WINDOWS */ - assert(0 <= tp->tv_usec && tp->tv_usec < SEC_TO_US); - return 0; -} - -void -_PyTime_gettimeofday(_PyTime_timeval *tp) -{ - if (pygettimeofday(tp, NULL, 0) < 0) { - /* cannot happen, _PyTime_Init() checks that pygettimeofday() works */ - assert(0); - tp->tv_sec = 0; - tp->tv_usec = 0; - } -} - static void error_time_t_overflow(void) { @@ -577,6 +477,20 @@ pygettimeofday_new(_PyTime_t *tp, _Py_clock_info_t *info, int raise) return 0; } +_PyTime_t +_PyTime_GetSystemClock(void) +{ + _PyTime_t t; + if (pygettimeofday_new(&t, NULL, 0) < 0) { + /* should not happen, _PyTime_Init() checked the clock at startup */ + assert(0); + + /* use a fixed value instead of a random value from the stack */ + t = 0; + } + return t; +} + int _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) { @@ -715,13 +629,8 @@ _PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) int _PyTime_Init(void) { - _PyTime_timeval tv; _PyTime_t t; - /* ensure that the system clock works */ - if (pygettimeofday(&tv, NULL, 1) < 0) - return -1; - /* ensure that the system clock works */ if (_PyTime_GetSystemClockWithInfo(&t, NULL) < 0) return -1; From 1bd18ba9a78c58b817564637f1937c2bc3920ecd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:25:38 +0200 Subject: [PATCH 02/20] Issue #22117: Cleanup pytime.c/.h --- Include/pytime.h | 74 ++++++++++++++++++++++++------------------------ Python/pytime.c | 12 ++++---- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 919ba30e5e5..b8727748a0f 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,13 +13,16 @@ functions and constants extern "C" { #endif -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; +#ifdef PY_INT64_T +/* _PyTime_t: Python timestamp with subsecond precision. It can be used to + store a duration, and so indirectly a date (related to another date, like + UNIX epoch). */ +typedef PY_INT64_T _PyTime_t; +#define _PyTime_MIN PY_LLONG_MIN +#define _PyTime_MAX PY_LLONG_MAX +#else +# error "_PyTime_t need signed 64-bit integer type" +#endif typedef enum { /* Round towards zero. */ @@ -32,12 +35,6 @@ typedef enum { _PyTime_ROUND_FLOOR } _PyTime_round_t; -/* Convert a number of seconds, int or float, to time_t. */ -PyAPI_FUNC(int) _PyTime_ObjectToTime_t( - PyObject *obj, - time_t *sec, - _PyTime_round_t); - /* Convert a time_t to a PyLong. */ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( time_t sec); @@ -46,6 +43,12 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); +/* Convert a number of seconds, int or float, to time_t. */ +PyAPI_FUNC(int) _PyTime_ObjectToTime_t( + PyObject *obj, + time_t *sec, + _PyTime_round_t); + /* Convert a number of seconds, int or float, to a timeval structure. usec is in the range [0; 999999] and rounded towards zero. For example, -1.2 is converted to (-2, 800000). */ @@ -64,22 +67,6 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( long *nsec, _PyTime_round_t); -/* Initialize time. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_Init(void); - -/****************** NEW _PyTime_t API **********************/ - -#ifdef PY_INT64_T -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef PY_INT64_T _PyTime_t; -#define _PyTime_MIN PY_LLONG_MIN -#define _PyTime_MAX PY_LLONG_MAX -#else -# error "_PyTime_t need signed 64-bit integer type" -#endif /* Create a timestamp from a number of nanoseconds (C long). */ PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(PY_LONG_LONG ns); @@ -125,14 +112,6 @@ PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); works. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); -/* Get the current time from the system clock. - * Fill clock information if info is not NULL. - * Raise an exception and return -1 on error, return 0 on success. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - /* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the difference between the @@ -142,6 +121,23 @@ PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( is available and works. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); + +/* Structure used by time.get_clock_info() */ +typedef struct { + const char *implementation; + int monotonic; + int adjustable; + double resolution; +} _Py_clock_info_t; + +/* Get the current time from the system clock. + * Fill clock information if info is not NULL. + * Raise an exception and return -1 on error, return 0 on success. + */ +PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + /* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the difference between the @@ -155,6 +151,10 @@ PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( _Py_clock_info_t *info); +/* Initialize time. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_Init(void); + #ifdef __cplusplus } #endif diff --git a/Python/pytime.c b/Python/pytime.c index 11e3a627ed9..d9ff3c63105 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -151,8 +151,6 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, return _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); } -/****************** NEW _PyTime_t API **********************/ - static void _PyTime_overflow(void) { @@ -161,7 +159,7 @@ _PyTime_overflow(void) } int -_PyTime_RoundTowardsInfinity(int is_neg, _PyTime_round_t round) +_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) { if (round == _PyTime_ROUND_FLOOR) return 0; @@ -196,7 +194,7 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) *tp = t; return res; } -#else +#elif !defined(MS_WINDOWS) static int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) { @@ -227,7 +225,7 @@ _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) d = PyFloat_AsDouble(obj); d *= 1e9; - if (_PyTime_RoundTowardsInfinity(d < 0, round)) + if (_PyTime_RoundTowardsPosInf(d < 0, round)) d = ceil(d); else d = floor(d); @@ -293,7 +291,7 @@ _PyTime_Multiply(_PyTime_t t, unsigned int multiply, _PyTime_round_t round) _PyTime_t k; if (multiply < SEC_TO_NS) { k = SEC_TO_NS / multiply; - if (_PyTime_RoundTowardsInfinity(t < 0, round)) + if (_PyTime_RoundTowardsPosInf(t < 0, round)) return (t + k - 1) / k; else return t / k; @@ -353,7 +351,7 @@ _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) res = -1; #endif - if (_PyTime_RoundTowardsInfinity(tv->tv_sec < 0, round)) + if (_PyTime_RoundTowardsPosInf(tv->tv_sec < 0, round)) tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); else tv->tv_usec = (int)(ns / US_TO_NS); From a3a100b594982f10911f2c9db6cf954bb8ff8f20 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 01:28:02 +0300 Subject: [PATCH 03/20] Issue #22390: test.regrtest now emits a warning if temporary files or directories are left after running a test. --- Lib/test/regrtest.py | 26 +++++++++++--------------- Misc/NEWS | 3 +++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b633631b00d..350e6849ee6 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1031,7 +1031,7 @@ def __init__(self, testname, verbose=0, quiet=False): # to a thread, so check processes first. 'multiprocessing.process._dangling', 'threading._dangling', 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'support.TESTFN', 'locale', 'warnings.showwarning', + 'files', 'locale', 'warnings.showwarning', ) def get_sys_argv(self): @@ -1187,20 +1187,16 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved): sysconfig._INSTALL_SCHEMES.clear() sysconfig._INSTALL_SCHEMES.update(saved[2]) - def get_support_TESTFN(self): - if os.path.isfile(support.TESTFN): - result = 'f' - elif os.path.isdir(support.TESTFN): - result = 'd' - else: - result = None - return result - def restore_support_TESTFN(self, saved_value): - if saved_value is None: - if os.path.isfile(support.TESTFN): - os.unlink(support.TESTFN) - elif os.path.isdir(support.TESTFN): - shutil.rmtree(support.TESTFN) + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] diff --git a/Misc/NEWS b/Misc/NEWS index 48d1801b0c1..b80cbe25f94 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -130,6 +130,9 @@ Library Tests ----- +- Issue #22390: test.regrtest now emits a warning if temporary files or + directories are left after running a test. + - Issue #23583: Added tests for standard IO streams in IDLE. Build From f81f0f9c63c8ae306d550b5bb387abf30e60a668 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 00:44:06 +0200 Subject: [PATCH 04/20] Issue #22117: Fix rounding and implement _PyTime_ROUND_FLOOR in: - _PyTime_ObjectToTime_t() - _PyTime_ObjectToTimespec() - _PyTime_ObjectToTimeval() --- Lib/test/test_time.py | 4 ++-- Python/pytime.c | 44 ++++++++++++++++++++----------------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index b0c97d5919f..bcbf41d0db8 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -647,13 +647,13 @@ def test_timespec(self): (1e-9, (0, 1), _PyTime.ROUND_DOWN), (1e-10, (0, 0), _PyTime.ROUND_DOWN), (-1e-9, (-1, 999999999), _PyTime.ROUND_DOWN), - (-1e-10, (-1, 999999999), _PyTime.ROUND_DOWN), + (-1e-10, (0, 0), _PyTime.ROUND_DOWN), (-1.2, (-2, 800000000), _PyTime.ROUND_DOWN), (0.9999999999, (0, 999999999), _PyTime.ROUND_DOWN), (1.1234567890, (1, 123456789), _PyTime.ROUND_DOWN), (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_DOWN), + (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), # Round away from zero (0, (0, 0), _PyTime.ROUND_UP), (-1, (-1, 0), _PyTime.ROUND_UP), diff --git a/Python/pytime.c b/Python/pytime.c index d9ff3c63105..2bf6ba5cf28 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -26,6 +26,14 @@ error_time_t_overflow(void) "timestamp out of range for platform time_t"); } +static int +_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) +{ + if (round == _PyTime_ROUND_FLOOR) + return 0; + return ((round == _PyTime_ROUND_UP) ^ is_neg); +} + time_t _PyLong_AsTime_t(PyObject *obj) { @@ -74,18 +82,16 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } floatpart *= denominator; - if (round == _PyTime_ROUND_UP) { - if (intpart >= 0) { - floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { - floatpart = floor(floatpart); + if (_PyTime_RoundTowardsPosInf(intpart < 0, round)) { + floatpart = ceil(floatpart); + if (floatpart >= denominator) { + floatpart = 0.0; + intpart += 1.0; } } + else { + floatpart = floor(floatpart); + } *sec = (time_t)intpart; err = intpart - (double)*sec; @@ -113,12 +119,10 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_UP) { - if (d >= 0) - d = ceil(d); - else - d = floor(d); - } + if (_PyTime_RoundTowardsPosInf(d < 0, round)) + d = ceil(d); + else + d = floor(d); (void)modf(d, &intpart); *sec = (time_t)intpart; @@ -158,14 +162,6 @@ _PyTime_overflow(void) "timestamp too large to convert to C _PyTime_t"); } -int -_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) -{ - if (round == _PyTime_ROUND_FLOOR) - return 0; - return ((round == _PyTime_ROUND_UP) ^ is_neg); -} - _PyTime_t _PyTime_FromNanoseconds(PY_LONG_LONG ns) { From dca028b86ade11441554f8cdb9d2ae56c119b413 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 01:02:57 +0200 Subject: [PATCH 05/20] Issue #22117: Fix os.utime(), it now rounds the timestamp towards minus infinity (-inf) instead of rounding towards zero. Replace _PyTime_ROUND_DOWN with _PyTime_ROUND_FLOOR. --- Misc/NEWS | 3 +++ Modules/posixmodule.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 96621a14786..e96d66e0ac0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- Issue #22117: Fix os.utime(), it now rounds the timestamp towards minus + infinity (-inf) instead of rounding towards zero. + - Issue #14260: The groupindex attribute of regular expression pattern object now is non-modifiable mapping. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9a44d469756..801305f535f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6127,9 +6127,9 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, PyObject *ns, } utime.now = 0; if (_PyTime_ObjectToTimespec(PyTuple_GET_ITEM(times, 0), - &a_sec, &a_nsec, _PyTime_ROUND_DOWN) == -1 || + &a_sec, &a_nsec, _PyTime_ROUND_FLOOR) == -1 || _PyTime_ObjectToTimespec(PyTuple_GET_ITEM(times, 1), - &m_sec, &m_nsec, _PyTime_ROUND_DOWN) == -1) { + &m_sec, &m_nsec, _PyTime_ROUND_FLOOR) == -1) { goto exit; } utime.atime_s = a_sec; From e4a994d6171f47ee9ba68ae1484d940349d62564 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 01:10:14 +0200 Subject: [PATCH 06/20] Issue #22117: Fix rounding of fromtimestamp() methods of datetime.datetime and datetime.time: round towards minus infinity ("floor") instead of rounding towards zero ("down"). --- Modules/_datetimemodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c3e54f7af43..ab2acae662e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2463,7 +2463,7 @@ date_local_from_object(PyObject *cls, PyObject *obj) struct tm *tm; time_t t; - if (_PyTime_ObjectToTime_t(obj, &t, _PyTime_ROUND_DOWN) == -1) + if (_PyTime_ObjectToTime_t(obj, &t, _PyTime_ROUND_FLOOR) == -1) return NULL; tm = localtime(&t); @@ -4095,7 +4095,8 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, time_t timet; long us; - if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_DOWN) == -1) + if (_PyTime_ObjectToTimeval(timestamp, + &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; assert(0 <= us && us <= 999999); From 160e819a1d0a01fe79b66bf398c925c0dac0ded1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:18:31 +0200 Subject: [PATCH 07/20] Issue #23694: Fix usage of _Py_open() in the _posixsubprocess module Don't call _Py_open() from _close_open_fds_safe() because it is call just after fork(). It's not good to play with locks (the GIL) between fork() and exec(). Use instead _Py_open_noraise() which doesn't touch to the GIL. --- Modules/_posixsubprocess.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index a33df211e98..0b385a1ab15 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -254,10 +254,9 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) { int fd_dir_fd; - fd_dir_fd = _Py_open(FD_DIR, O_RDONLY); + fd_dir_fd = _Py_open_noraise(FD_DIR, O_RDONLY); if (fd_dir_fd == -1) { /* No way to get a list of open fds. */ - PyErr_Clear(); _close_fds_by_brute_force(start_fd, py_fds_to_keep); return; } else { From ea9c0dd2c27884691f0a0af983fd41d4d818e93f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:51:13 +0200 Subject: [PATCH 08/20] Issue #22117: Fix usage of _PyTime_AsTimeval() Add _PyTime_AsTimeval_noraise() function. Call it when it's not possible (or not useful) to raise a Python exception on overflow. --- Include/pytime.h | 8 +++++++- Modules/_ssl.c | 4 +--- Modules/_testcapimodule.c | 5 +---- Modules/socketmodule.c | 8 ++------ Modules/timemodule.c | 5 +---- Python/pytime.c | 19 +++++++++++++++++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index b8727748a0f..f14e1fc6fde 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -94,11 +94,17 @@ PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); /* Convert a timestamp to a timeval structure (microsecond resolution). tv_usec is always positive. - Return -1 if the conversion overflowed, return 0 on success. */ + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); +/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */ +PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) /* Convert a timestamp to a timespec structure (nanosecond resolution). tv_nsec is always positive. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 54f5d140ed4..217c77c5933 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1651,9 +1651,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) if (!_PyIsSelectable_fd(s->sock_fd)) return SOCKET_TOO_LARGE_FOR_SELECT; - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9abb7ccd195..128ba094b2e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3427,11 +3427,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) return NULL; t = _PyTime_FromNanoseconds(ns); - if (_PyTime_AsTimeval(t, &tv, round) < 0) { - PyErr_SetString(PyExc_OverflowError, - "timeout doesn't fit into C timeval"); + if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - } seconds = PyLong_FromLong((PY_LONG_LONG)tv.tv_sec); if (seconds == NULL) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 93dcd419481..513405e507a 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -641,9 +641,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) n = poll(&pollfd, 1, timeout_int); Py_END_ALLOW_THREADS; #else - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(interval, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_UP); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); @@ -2454,9 +2452,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, struct timeval tv; int conv; - /* conversion was already checked for overflow when - the timeout was set */ - (void)_PyTime_AsTimeval(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); Py_BEGIN_ALLOW_THREADS FD_ZERO(&fds); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 99e83cc6dc0..3ed3fb31bdc 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1405,11 +1405,8 @@ pysleep(_PyTime_t secs) do { #ifndef MS_WINDOWS - if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) { - PyErr_SetString(PyExc_OverflowError, - "delay doesn't fit into C timeval"); + if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) return -1; - } Py_BEGIN_ALLOW_THREADS err = select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &timeout); diff --git a/Python/pytime.c b/Python/pytime.c index 2bf6ba5cf28..a7eda869a07 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -311,8 +311,9 @@ _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) return _PyTime_Multiply(t, 1000 * 1000, round); } -int -_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +static int +_PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, + int raise) { _PyTime_t secs, ns; int res = 0; @@ -357,9 +358,23 @@ _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) tv->tv_sec += 1; } + if (res && raise) + _PyTime_overflow(); return res; } +int +_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimeval_impl(t, tv, round, 1); +} + +int +_PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimeval_impl(t, tv, round, 0); +} + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) int _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) From edddf991d9ea88d16c8ebef7a5f829822d8025fa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 02:54:57 +0200 Subject: [PATCH 09/20] Issue #22117: Add assertions to _PyTime_AsTimeval() and _PyTime_AsTimespec() to check that microseconds and nanoseconds fits into the specified range. --- Python/pytime.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/pytime.c b/Python/pytime.c index a7eda869a07..0d28911c378 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -360,6 +360,8 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, if (res && raise) _PyTime_overflow(); + + assert(0 <= tv->tv_usec && tv->tv_usec <= 999999); return res; } @@ -393,6 +395,8 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) return -1; } ts->tv_nsec = nsec; + + assert(0 <= ts->tv_nsec && ts->tv_nsec <= 999999999); return 0; } #endif From 520bddf79a06c06d7b293408a6e3cc119f808afb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:21:06 +0200 Subject: [PATCH 10/20] Issue #23752: When built from an existing file descriptor, io.FileIO() now only calls fstat() once. Before fstat() was called twice, which was not necessary. --- Misc/NEWS | 26 ++++++++++++++++++++++++++ Modules/_io/fileio.c | 24 ------------------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e96d66e0ac0..2b9ce2e7443 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,32 @@ Python News +++++++++++ +What's New in Python 3.5.0 alpha 4? +=================================== + +Release date: XXX + +Core and Builtins +----------------- + +Library +------- + +- Issue #23752: When built from an existing file descriptor, io.FileIO() now + only calls fstat() once. Before fstat() was called twice, which was not + necessary. + +Build +----- + +Tests +----- + +Tools/Demos +----------- + + + What's New in Python 3.5.0 alpha 3? =================================== diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b35a51b4e1a..595f99ee858 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -177,28 +177,6 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; } -static int -check_fd(int fd) -{ - struct _Py_stat_struct buf; - if (_Py_fstat(fd, &buf) < 0 && -#ifdef MS_WINDOWS - GetLastError() == ERROR_INVALID_HANDLE -#else - errno == EBADF -#endif - ) { - PyObject *exc; - char *msg = strerror(EBADF); - exc = PyObject_CallFunction(PyExc_OSError, "(is)", - EBADF, msg); - PyErr_SetObject(PyExc_OSError, exc); - Py_XDECREF(exc); - return -1; - } - return 0; -} - #ifdef O_CLOEXEC extern int _Py_open_cloexec_works; #endif @@ -355,8 +333,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) #endif if (fd >= 0) { - if (check_fd(fd)) - goto error; self->fd = fd; self->closefd = closefd; } From bcdd777d3c01a6db3b4357922663624ef617e65a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:52:49 +0200 Subject: [PATCH 11/20] Issue #22117: Add _PyTime_ROUND_CEILING rounding method for timestamps Add also more tests for ROUNd_FLOOR. --- Include/pytime.h | 5 ++- Lib/test/test_time.py | 65 ++++++++++++++++++++++++++++++++++++--- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 2 ++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index f14e1fc6fde..ff3e203e359 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -32,7 +32,10 @@ typedef enum { _PyTime_ROUND_UP, /* Round towards minus infinity (-inf). For example, used to read a clock. */ - _PyTime_ROUND_FLOOR + _PyTime_ROUND_FLOOR, + /* Round towards infinity (+inf). + For example, used for timeout to wait "at least" N seconds. */ + _PyTime_ROUND_CEILING } _PyTime_round_t; /* Convert a time_t to a PyLong. */ diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bcbf41d0db8..472110c687a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -28,13 +28,16 @@ class _PyTime(enum.IntEnum): ROUND_DOWN = 0 # Round away from zero ROUND_UP = 1 - # Round towards -Infinity + # Round towards minus infinity (-inf) ROUND_FLOOR = 2 + # Round towards infinity (+inf) + ROUND_CEILING = 3 ALL_ROUNDING_METHODS = ( _PyTime.ROUND_UP, _PyTime.ROUND_DOWN, - _PyTime.ROUND_FLOOR) + _PyTime.ROUND_FLOOR, + _PyTime.ROUND_CEILING) class TimeTestCase(unittest.TestCase): @@ -621,6 +624,13 @@ def test_time_t(self): (-1.9, -1, _PyTime.ROUND_DOWN), (1.0, 1, _PyTime.ROUND_DOWN), (1.9, 1, _PyTime.ROUND_DOWN), + # Round towards minus infinity (-inf) + (0, 0, _PyTime.ROUND_FLOOR), + (-1, -1, _PyTime.ROUND_FLOOR), + (-1.0, -1, _PyTime.ROUND_FLOOR), + (-1.9, -2, _PyTime.ROUND_FLOOR), + (1.0, 1, _PyTime.ROUND_FLOOR), + (1.9, 1, _PyTime.ROUND_FLOOR), # Round away from zero (0, 0, _PyTime.ROUND_UP), (-1, -1, _PyTime.ROUND_UP), @@ -628,10 +638,17 @@ def test_time_t(self): (-1.9, -2, _PyTime.ROUND_UP), (1.0, 1, _PyTime.ROUND_UP), (1.9, 2, _PyTime.ROUND_UP), + # Round towards infinity (+inf) + (0, 0, _PyTime.ROUND_CEILING), + (-1, -1, _PyTime.ROUND_CEILING), + (-1.0, -1, _PyTime.ROUND_CEILING), + (-1.9, -1, _PyTime.ROUND_CEILING), + (1.0, 1, _PyTime.ROUND_CEILING), + (1.9, 2, _PyTime.ROUND_CEILING), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - rnd = _PyTime.ROUND_DOWN + rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_time_t, invalid, rnd) @@ -654,6 +671,20 @@ def test_timespec(self): (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), + # Round towards minus infinity (-inf) + (0, (0, 0), _PyTime.ROUND_FLOOR), + (-1, (-1, 0), _PyTime.ROUND_FLOOR), + (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), + (1e-9, (0, 1), _PyTime.ROUND_FLOOR), + (1e-10, (0, 0), _PyTime.ROUND_FLOOR), + (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), + (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), + (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), + (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), + (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), + (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), + (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), + (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), # Round away from zero (0, (0, 0), _PyTime.ROUND_UP), (-1, (-1, 0), _PyTime.ROUND_UP), @@ -668,11 +699,25 @@ def test_timespec(self): (1.1234567899, (1, 123456790), _PyTime.ROUND_UP), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_UP), (-1.1234567891, (-2, 876543210), _PyTime.ROUND_UP), + # Round towards infinity (+inf) + (0, (0, 0), _PyTime.ROUND_CEILING), + (-1, (-1, 0), _PyTime.ROUND_CEILING), + (-1.0, (-1, 0), _PyTime.ROUND_CEILING), + (1e-9, (0, 1), _PyTime.ROUND_CEILING), + (1e-10, (0, 1), _PyTime.ROUND_CEILING), + (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), + (-1e-10, (0, 0), _PyTime.ROUND_CEILING), + (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), + (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), + (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), + (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), + (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), + (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - rnd = _PyTime.ROUND_DOWN + rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, pytime_object_to_timespec, invalid, rnd) @@ -794,27 +839,34 @@ def test_FromSecondsObject(self): UP = _PyTime.ROUND_UP DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING for obj, ts, rnd in ( # close to zero + ( 1e-10, 1, CEILING), ( 1e-10, 1, UP), ( 1e-10, 0, DOWN), ( 1e-10, 0, FLOOR), + (-1e-10, 0, CEILING), (-1e-10, 0, DOWN), (-1e-10, -1, UP), (-1e-10, -1, FLOOR), # test rounding of the last nanosecond + ( 1.1234567899, 1123456790, CEILING), ( 1.1234567899, 1123456790, UP), ( 1.1234567899, 1123456789, DOWN), ( 1.1234567899, 1123456789, FLOOR), + (-1.1234567899, -1123456789, CEILING), (-1.1234567899, -1123456789, DOWN), (-1.1234567899, -1123456790, UP), (-1.1234567899, -1123456790, FLOOR), # close to 1 second + ( 0.9999999999, 1000000000, CEILING), ( 0.9999999999, 1000000000, UP), ( 0.9999999999, 999999999, DOWN), ( 0.9999999999, 999999999, FLOOR), + (-0.9999999999, -999999999, CEILING), (-0.9999999999, -999999999, DOWN), (-0.9999999999, -1000000000, UP), (-0.9999999999, -1000000000, FLOOR), @@ -890,19 +942,24 @@ def test_timeval(self): UP = _PyTime.ROUND_UP DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING for ns, tv, rnd in ( # nanoseconds + (1, (0, 1), CEILING), (1, (0, 1), UP), (1, (0, 0), DOWN), (1, (0, 0), FLOOR), + (-1, (0, 0), CEILING), (-1, (0, 0), DOWN), (-1, (-1, 999999), UP), (-1, (-1, 999999), FLOOR), # seconds + nanoseconds + (1234567001, (1, 234568), CEILING), (1234567001, (1, 234568), UP), (1234567001, (1, 234567), DOWN), (1234567001, (1, 234567), FLOOR), + (-1234567001, (-2, 765433), CEILING), (-1234567001, (-2, 765433), DOWN), (-1234567001, (-2, 765432), UP), (-1234567001, (-2, 765432), FLOOR), diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 128ba094b2e..25f916beb9e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2635,7 +2635,7 @@ static int check_time_rounding(int round) { if (round != _PyTime_ROUND_DOWN && round != _PyTime_ROUND_UP - && round != _PyTime_ROUND_FLOOR) { + && round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index 0d28911c378..ca4386aa089 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -31,6 +31,8 @@ _PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) { if (round == _PyTime_ROUND_FLOOR) return 0; + if (round == _PyTime_ROUND_CEILING) + return 1; return ((round == _PyTime_ROUND_UP) ^ is_neg); } From 869e1778c0bcfc0928701c6ae0703934359d036b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:49:14 +0200 Subject: [PATCH 12/20] Issue #22117: Replace usage of _PyTime_ROUND_UP with _PyTime_ROUND_CEILING All these functions only accept positive timeouts, so this change has no effect in practice. --- Modules/_ssl.c | 4 ++-- Modules/_threadmodule.c | 7 ++++--- Modules/selectmodule.c | 7 ++++--- Modules/signalmodule.c | 3 ++- Modules/socketmodule.c | 13 +++++++------ Modules/timemodule.c | 6 +++--- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 217c77c5933..3c909a6a3a5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1637,7 +1637,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) /* s->sock_timeout is in seconds, timeout in ms */ timeout = (int)_PyTime_AsMilliseconds(s->sock_timeout, - _PyTime_ROUND_UP); + _PyTime_ROUND_CEILING); PySSL_BEGIN_ALLOW_THREADS rc = poll(&pollfd, 1, timeout); @@ -1651,7 +1651,7 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) if (!_PyIsSelectable_fd(s->sock_fd)) return SOCKET_TOO_LARGE_FOR_SELECT; - _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_CEILING); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 07b01f09c4d..0907aa0eb25 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -59,7 +59,7 @@ acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) endtime = _PyTime_GetMonotonicClock() + timeout; do { - microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_UP); + microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); /* first a simple non-blocking try without releasing the GIL */ r = PyThread_acquire_lock_timed(lock, 0, 0); @@ -110,7 +110,8 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, return -1; if (timeout_obj - && _PyTime_FromSecondsObject(timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + && _PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return -1; if (!blocking && *timeout != unset_timeout ) { @@ -128,7 +129,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, else if (*timeout != unset_timeout) { _PyTime_t microseconds; - microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_UP); + microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_CEILING); if (microseconds >= PY_TIMEOUT_MAX) { PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 2c82ce7549b..a8523440a9a 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -209,13 +209,13 @@ select_select(PyObject *self, PyObject *args) else { _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, tout, _PyTime_ROUND_UP) < 0) { + if (_PyTime_FromSecondsObject(&ts, tout, _PyTime_ROUND_CEILING) < 0) { PyErr_SetString(PyExc_TypeError, "timeout must be a float or None"); return NULL; } - if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_UP) == -1) + if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_CEILING) == -1) return NULL; if (tv.tv_sec < 0) { PyErr_SetString(PyExc_ValueError, "timeout must be non-negative"); @@ -2014,7 +2014,8 @@ kqueue_queue_control(kqueue_queue_Object *self, PyObject *args) else { _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, otimeout, _PyTime_ROUND_UP) < 0) { + if (_PyTime_FromSecondsObject(&ts, + otimeout, _PyTime_ROUND_CEILING) < 0) { PyErr_Format(PyExc_TypeError, "timeout argument must be an number " "or None, got %.200s", diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index f3b2e298678..1b3589d7c65 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -977,7 +977,8 @@ signal_sigtimedwait(PyObject *self, PyObject *args) &signals, &timeout_obj)) return NULL; - if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + if (_PyTime_FromSecondsObject(&timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; if (timeout < 0) { diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 513405e507a..a6c47ae5269 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -633,7 +633,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) pollfd.events = writing ? POLLOUT : POLLIN; /* s->sock_timeout is in seconds, timeout in ms */ - timeout = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_UP); + timeout = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); assert(timeout <= INT_MAX); timeout_int = (int)timeout; @@ -641,7 +641,7 @@ internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) n = poll(&pollfd, 1, timeout_int); Py_END_ALLOW_THREADS; #else - _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_CEILING); FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); @@ -2191,7 +2191,8 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) return 0; } - if (_PyTime_FromSecondsObject(timeout, timeout_obj, _PyTime_ROUND_UP) < 0) + if (_PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_CEILING) < 0) return -1; if (*timeout < 0) { @@ -2200,10 +2201,10 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) } #ifdef MS_WINDOWS - overflow = (_PyTime_AsTimeval(timeout, &tv, _PyTime_ROUND_UP) < 0); + overflow = (_PyTime_AsTimeval(timeout, &tv, _PyTime_ROUND_CEILING) < 0); #endif #ifndef HAVE_POLL - timeout = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_UP); + timeout = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); overflow = (timeout > INT_MAX); #endif if (overflow) { @@ -2452,7 +2453,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, struct timeval tv; int conv; - _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_UP); + _PyTime_AsTimeval_noraise(s->sock_timeout, &tv, _PyTime_ROUND_CEILING); Py_BEGIN_ALLOW_THREADS FD_ZERO(&fds); diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3ed3fb31bdc..74c544a62dd 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -221,7 +221,7 @@ static PyObject * time_sleep(PyObject *self, PyObject *obj) { _PyTime_t secs; - if (_PyTime_FromSecondsObject(&secs, obj, _PyTime_ROUND_UP)) + if (_PyTime_FromSecondsObject(&secs, obj, _PyTime_ROUND_CEILING)) return NULL; if (secs < 0) { PyErr_SetString(PyExc_ValueError, @@ -1405,7 +1405,7 @@ pysleep(_PyTime_t secs) do { #ifndef MS_WINDOWS - if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_UP) < 0) + if (_PyTime_AsTimeval(secs, &timeout, _PyTime_ROUND_CEILING) < 0) return -1; Py_BEGIN_ALLOW_THREADS @@ -1420,7 +1420,7 @@ pysleep(_PyTime_t secs) return -1; } #else - millisecs = _PyTime_AsMilliseconds(secs, _PyTime_ROUND_UP); + millisecs = _PyTime_AsMilliseconds(secs, _PyTime_ROUND_CEILING); if (millisecs > (double)ULONG_MAX) { PyErr_SetString(PyExc_OverflowError, "sleep length is too large"); From 7d0325d6c811d66d98955e9fbf9a71c50f7e770c Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 29 Mar 2015 21:53:05 -0400 Subject: [PATCH 13/20] #23745: handle duplicate MIME parameter names in new parser. This mimics get_param's error handling for the most part. It is slightly better in some regards as get_param can produce some really weird results for duplicate *0* parts. It departs from get_param slightly in that if we have a mix of non-extended and extended pieces for the same parameter name, the new parser assumes they were all supposed to be extended and concatenates all the values, whereas get_param always picks the non-extended parameter value. All of this error recovery is pretty much arbitrary decisions... --- Lib/email/_header_value_parser.py | 34 ++++-- .../test_email/test__header_value_parser.py | 109 ++++++++++++++++++ Misc/NEWS | 3 + 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 1806cac9053..a9bdf4458b6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -71,6 +71,7 @@ import urllib # For urllib.parse.unquote from string import hexdigits from collections import OrderedDict +from operator import itemgetter from email import _encoded_words as _ew from email import errors from email import utils @@ -1098,15 +1099,34 @@ def params(self): params[name] = [] params[name].append((token.section_number, token)) for name, parts in params.items(): - parts = sorted(parts) - # XXX: there might be more recovery we could do here if, for - # example, this is really a case of a duplicate attribute name. + parts = sorted(parts, key=itemgetter(0)) + first_param = parts[0][1] + charset = first_param.charset + # Our arbitrary error recovery is to ignore duplicate parameters, + # to use appearance order if there are duplicate rfc 2231 parts, + # and to ignore gaps. This mimics the error recovery of get_param. + if not first_param.extended and len(parts) > 1: + if parts[1][0] == 0: + parts[1][1].defects.append(errors.InvalidHeaderDefect( + 'duplicate parameter name; duplicate(s) ignored')) + parts = parts[:1] + # Else assume the *0* was missing...note that this is different + # from get_param, but we registered a defect for this earlier. value_parts = [] - charset = parts[0][1].charset - for i, (section_number, param) in enumerate(parts): + i = 0 + for section_number, param in parts: if section_number != i: - param.defects.append(errors.InvalidHeaderDefect( - "inconsistent multipart parameter numbering")) + # We could get fancier here and look for a complete + # duplicate extended parameter and ignore the second one + # seen. But we're not doing that. The old code didn't. + if not param.extended: + param.defects.append(errors.InvalidHeaderDefect( + 'duplicate parameter name; duplicate ignored')) + continue + else: + param.defects.append(errors.InvalidHeaderDefect( + "inconsistent RFC2231 parameter numbering")) + i += 1 value = param.param_value if param.extended: try: diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 5404d1913f8..d028f7440be 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2456,6 +2456,115 @@ def test_invalid_content_transfer_encoding(self): ";foo", ";foo", ";foo", [errors.InvalidHeaderDefect]*3 ) + +@parameterize +class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): + + def mime_parameters_as_value(self, + value, + tl_str, + tl_value, + params, + defects): + mime_parameters = self._test_parse_x(parser.parse_mime_parameters, + value, tl_str, tl_value, defects) + self.assertEqual(mime_parameters.token_type, 'mime-parameters') + self.assertEqual(list(mime_parameters.params), params) + + + mime_parameters_params = { + + 'simple': ( + 'filename="abc.py"', + ' filename="abc.py"', + 'filename=abc.py', + [('filename', 'abc.py')], + []), + + 'multiple_keys': ( + 'filename="abc.py"; xyz=abc', + ' filename="abc.py"; xyz="abc"', + 'filename=abc.py; xyz=abc', + [('filename', 'abc.py'), ('xyz', 'abc')], + []), + + 'split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', '201.tif')], + []), + + # Note that it is undefined what we should do for error recovery when + # there are duplicate parameter names or duplicate parts in a split + # part. We choose to ignore all duplicate parameters after the first + # and to take duplicate or missing rfc 2231 parts in apperance order. + # This is backward compatible with get_param's behavior, but the + # decisions are arbitrary. + + 'duplicate_key': ( + 'filename=abc.gif; filename=def.tiff', + ' filename="abc.gif"', + "filename=abc.gif; filename=def.tiff", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename=abc.gif", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_key_with_split_value_other_order': ( + "filename=abc.gif; " + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + ' filename="abc.gif"', + "filename=abc.gif;" + " filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66", + [('filename', 'abc.gif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_in_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*1*=%74%69%66;" + " filename*1*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]), + + 'missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + ' filename="201.tif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;", + [('filename', '201.tif')], + [errors.InvalidHeaderDefect]), + + 'duplicate_and_missing_split_value': ( + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + ' filename="201.tifabc.gif"', + "filename*0*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66;" + " filename*3*=abc.gif", + [('filename', '201.tifabc.gif')], + [errors.InvalidHeaderDefect]*2), + + # Here we depart from get_param and assume the *0* was missing. + 'duplicate_with_broken_split_value': ( + "filename=abc.gif; " + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + ' filename="abc.gif201.tif"', + "filename=abc.gif;" + " filename*2*=iso-8859-1''%32%30%31%2E; filename*3*=%74%69%66", + [('filename', 'abc.gif201.tif')], + # Defects are apparent missing *0*, and two 'out of sequence'. + [errors.InvalidHeaderDefect]*3), + + } + @parameterize class Test_parse_mime_version(TestParserMixin, TestEmailBase): diff --git a/Misc/NEWS b/Misc/NEWS index b80cbe25f94..ebac8d52f65 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,9 @@ Core and Builtins Library ------- +- Issue #23745: The new email header parser now handles duplicate MIME + parameter names without error, similar to how get_param behaves. + - Issue #23792: Ignore KeyboardInterrupt when the pydoc pager is active. This mimics the behavior of the standard unix pagers, and prevents pipepager from shutting down while the pager itself is still running. From a695f83f0de060a77352174be8a5c6f6500ab98a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 03:57:14 +0200 Subject: [PATCH 14/20] Issue #22117: Remove _PyTime_ROUND_DOWN and _PyTime_ROUND_UP rounding methods Use _PyTime_ROUND_FLOOR and _PyTime_ROUND_CEILING instead. --- Include/pytime.h | 7 +-- Lib/test/test_time.py | 100 +++++--------------------------------- Modules/_testcapimodule.c | 3 +- Python/pytime.c | 20 ++------ 4 files changed, 20 insertions(+), 110 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index ff3e203e359..1f14d6d4beb 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -25,14 +25,9 @@ typedef PY_INT64_T _PyTime_t; #endif typedef enum { - /* Round towards zero. */ - _PyTime_ROUND_DOWN=0, - /* Round away from zero. - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_UP, /* Round towards minus infinity (-inf). For example, used to read a clock. */ - _PyTime_ROUND_FLOOR, + _PyTime_ROUND_FLOOR=0, /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ _PyTime_ROUND_CEILING diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 472110c687a..4747cc6ea0d 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -24,20 +24,12 @@ SEC_TO_NS = 10 ** 9 class _PyTime(enum.IntEnum): - # Round towards zero - ROUND_DOWN = 0 - # Round away from zero - ROUND_UP = 1 # Round towards minus infinity (-inf) - ROUND_FLOOR = 2 + ROUND_FLOOR = 0 # Round towards infinity (+inf) - ROUND_CEILING = 3 + ROUND_CEILING = 1 -ALL_ROUNDING_METHODS = ( - _PyTime.ROUND_UP, - _PyTime.ROUND_DOWN, - _PyTime.ROUND_FLOOR, - _PyTime.ROUND_CEILING) +ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) class TimeTestCase(unittest.TestCase): @@ -617,13 +609,6 @@ def setUp(self): def test_time_t(self): from _testcapi import pytime_object_to_time_t for obj, time_t, rnd in ( - # Round towards zero - (0, 0, _PyTime.ROUND_DOWN), - (-1, -1, _PyTime.ROUND_DOWN), - (-1.0, -1, _PyTime.ROUND_DOWN), - (-1.9, -1, _PyTime.ROUND_DOWN), - (1.0, 1, _PyTime.ROUND_DOWN), - (1.9, 1, _PyTime.ROUND_DOWN), # Round towards minus infinity (-inf) (0, 0, _PyTime.ROUND_FLOOR), (-1, -1, _PyTime.ROUND_FLOOR), @@ -631,13 +616,6 @@ def test_time_t(self): (-1.9, -2, _PyTime.ROUND_FLOOR), (1.0, 1, _PyTime.ROUND_FLOOR), (1.9, 1, _PyTime.ROUND_FLOOR), - # Round away from zero - (0, 0, _PyTime.ROUND_UP), - (-1, -1, _PyTime.ROUND_UP), - (-1.0, -1, _PyTime.ROUND_UP), - (-1.9, -2, _PyTime.ROUND_UP), - (1.0, 1, _PyTime.ROUND_UP), - (1.9, 2, _PyTime.ROUND_UP), # Round towards infinity (+inf) (0, 0, _PyTime.ROUND_CEILING), (-1, -1, _PyTime.ROUND_CEILING), @@ -657,20 +635,6 @@ def test_time_t(self): def test_timespec(self): from _testcapi import pytime_object_to_timespec for obj, timespec, rnd in ( - # Round towards zero - (0, (0, 0), _PyTime.ROUND_DOWN), - (-1, (-1, 0), _PyTime.ROUND_DOWN), - (-1.0, (-1, 0), _PyTime.ROUND_DOWN), - (1e-9, (0, 1), _PyTime.ROUND_DOWN), - (1e-10, (0, 0), _PyTime.ROUND_DOWN), - (-1e-9, (-1, 999999999), _PyTime.ROUND_DOWN), - (-1e-10, (0, 0), _PyTime.ROUND_DOWN), - (-1.2, (-2, 800000000), _PyTime.ROUND_DOWN), - (0.9999999999, (0, 999999999), _PyTime.ROUND_DOWN), - (1.1234567890, (1, 123456789), _PyTime.ROUND_DOWN), - (1.1234567899, (1, 123456789), _PyTime.ROUND_DOWN), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_DOWN), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_DOWN), # Round towards minus infinity (-inf) (0, (0, 0), _PyTime.ROUND_FLOOR), (-1, (-1, 0), _PyTime.ROUND_FLOOR), @@ -685,20 +649,6 @@ def test_timespec(self): (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), - # Round away from zero - (0, (0, 0), _PyTime.ROUND_UP), - (-1, (-1, 0), _PyTime.ROUND_UP), - (-1.0, (-1, 0), _PyTime.ROUND_UP), - (1e-9, (0, 1), _PyTime.ROUND_UP), - (1e-10, (0, 1), _PyTime.ROUND_UP), - (-1e-9, (-1, 999999999), _PyTime.ROUND_UP), - (-1e-10, (-1, 999999999), _PyTime.ROUND_UP), - (-1.2, (-2, 800000000), _PyTime.ROUND_UP), - (0.9999999999, (1, 0), _PyTime.ROUND_UP), - (1.1234567890, (1, 123456790), _PyTime.ROUND_UP), - (1.1234567899, (1, 123456790), _PyTime.ROUND_UP), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_UP), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_UP), # Round towards infinity (+inf) (0, (0, 0), _PyTime.ROUND_CEILING), (-1, (-1, 0), _PyTime.ROUND_CEILING), @@ -836,40 +786,26 @@ def test_FromSecondsObject(self): PyTime_FromSecondsObject(-9223372037.0, rnd) # Conversion giving different results depending on the rounding method - UP = _PyTime.ROUND_UP - DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING for obj, ts, rnd in ( # close to zero - ( 1e-10, 1, CEILING), - ( 1e-10, 1, UP), - ( 1e-10, 0, DOWN), ( 1e-10, 0, FLOOR), - (-1e-10, 0, CEILING), - (-1e-10, 0, DOWN), - (-1e-10, -1, UP), + ( 1e-10, 1, CEILING), (-1e-10, -1, FLOOR), + (-1e-10, 0, CEILING), # test rounding of the last nanosecond - ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, UP), - ( 1.1234567899, 1123456789, DOWN), ( 1.1234567899, 1123456789, FLOOR), - (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456789, DOWN), - (-1.1234567899, -1123456790, UP), + ( 1.1234567899, 1123456790, CEILING), (-1.1234567899, -1123456790, FLOOR), + (-1.1234567899, -1123456789, CEILING), # close to 1 second - ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, UP), - ( 0.9999999999, 999999999, DOWN), ( 0.9999999999, 999999999, FLOOR), - (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -999999999, DOWN), - (-0.9999999999, -1000000000, UP), + ( 0.9999999999, 1000000000, CEILING), (-0.9999999999, -1000000000, FLOOR), + (-0.9999999999, -999999999, CEILING), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -939,30 +875,20 @@ def test_timeval(self): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - UP = _PyTime.ROUND_UP - DOWN = _PyTime.ROUND_DOWN FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING for ns, tv, rnd in ( # nanoseconds - (1, (0, 1), CEILING), - (1, (0, 1), UP), - (1, (0, 0), DOWN), (1, (0, 0), FLOOR), - (-1, (0, 0), CEILING), - (-1, (0, 0), DOWN), - (-1, (-1, 999999), UP), + (1, (0, 1), CEILING), (-1, (-1, 999999), FLOOR), + (-1, (0, 0), CEILING), # seconds + nanoseconds - (1234567001, (1, 234568), CEILING), - (1234567001, (1, 234568), UP), - (1234567001, (1, 234567), DOWN), (1234567001, (1, 234567), FLOOR), - (-1234567001, (-2, 765433), CEILING), - (-1234567001, (-2, 765433), DOWN), - (-1234567001, (-2, 765432), UP), + (1234567001, (1, 234568), CEILING), (-1234567001, (-2, 765432), FLOOR), + (-1234567001, (-2, 765433), CEILING), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 25f916beb9e..253efb64366 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2634,8 +2634,7 @@ run_in_subinterp(PyObject *self, PyObject *args) static int check_time_rounding(int round) { - if (round != _PyTime_ROUND_DOWN && round != _PyTime_ROUND_UP - && round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { + if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index ca4386aa089..98f29acc38b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -26,16 +26,6 @@ error_time_t_overflow(void) "timestamp out of range for platform time_t"); } -static int -_PyTime_RoundTowardsPosInf(int is_neg, _PyTime_round_t round) -{ - if (round == _PyTime_ROUND_FLOOR) - return 0; - if (round == _PyTime_ROUND_CEILING) - return 1; - return ((round == _PyTime_ROUND_UP) ^ is_neg); -} - time_t _PyLong_AsTime_t(PyObject *obj) { @@ -84,7 +74,7 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } floatpart *= denominator; - if (_PyTime_RoundTowardsPosInf(intpart < 0, round)) { + if (round == _PyTime_ROUND_CEILING) { floatpart = ceil(floatpart); if (floatpart >= denominator) { floatpart = 0.0; @@ -121,7 +111,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (_PyTime_RoundTowardsPosInf(d < 0, round)) + if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -223,7 +213,7 @@ _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) d = PyFloat_AsDouble(obj); d *= 1e9; - if (_PyTime_RoundTowardsPosInf(d < 0, round)) + if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -289,7 +279,7 @@ _PyTime_Multiply(_PyTime_t t, unsigned int multiply, _PyTime_round_t round) _PyTime_t k; if (multiply < SEC_TO_NS) { k = SEC_TO_NS / multiply; - if (_PyTime_RoundTowardsPosInf(t < 0, round)) + if (round == _PyTime_ROUND_CEILING) return (t + k - 1) / k; else return t / k; @@ -350,7 +340,7 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (_PyTime_RoundTowardsPosInf(tv->tv_sec < 0, round)) + if (round == _PyTime_ROUND_CEILING) tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); else tv->tv_usec = (int)(ns / US_TO_NS); From 7901b48a1f89b9bfa9d111ae3725400b466a9baa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:09:54 +0300 Subject: [PATCH 15/20] Issue #23171: csv.Writer.writerow() now supports arbitrary iterables. --- Doc/library/csv.rst | 4 ++- Lib/csv.py | 7 ++-- Lib/test/test_csv.py | 8 +++++ Misc/NEWS | 2 ++ Modules/_csv.c | 79 ++++++++++++++++++++++---------------------- 5 files changed, 54 insertions(+), 46 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index e7516b67f92..325a4219938 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -419,7 +419,7 @@ Writer Objects :class:`Writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -a sequence of strings or numbers for :class:`Writer` objects and a dictionary +an iterable of strings or numbers for :class:`Writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which @@ -431,6 +431,8 @@ read CSV files (assuming they support complex numbers at all). Write the *row* parameter to the writer's file object, formatted according to the current dialect. + .. versionchanged:: 3.5 + Added support of arbitrary iterables. .. method:: csvwriter.writerows(rows) diff --git a/Lib/csv.py b/Lib/csv.py index c3c31f01fd0..ca40e5e0efc 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -147,16 +147,13 @@ def _dict_to_list(self, rowdict): if wrong_fields: raise ValueError("dict contains fields not in fieldnames: " + ", ".join([repr(x) for x in wrong_fields])) - return [rowdict.get(key, self.restval) for key in self.fieldnames] + return (rowdict.get(key, self.restval) for key in self.fieldnames) def writerow(self, rowdict): return self.writer.writerow(self._dict_to_list(rowdict)) def writerows(self, rowdicts): - rows = [] - for rowdict in rowdicts: - rows.append(self._dict_to_list(rowdict)) - return self.writer.writerows(rows) + return self.writer.writerows(map(self._dict_to_list, rowdicts)) # Guard Sniffer's type checking against builds that exclude complex() try: diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 41ef790eb25..7be3cc3c06d 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -186,6 +186,14 @@ def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,p\\,q', escapechar='\\', quoting = csv.QUOTE_NONE) + def test_write_iterable(self): + self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') + self._write_test(iter(['a', 1, None]), 'a,1,') + self._write_test(iter([]), '') + self._write_test(iter([None]), '""') + self._write_error_test(csv.Error, iter([None]), quoting=csv.QUOTE_NONE) + self._write_test(iter([None, None]), ',') + def test_writerows(self): class BrokenFile: def write(self, buf): diff --git a/Misc/NEWS b/Misc/NEWS index ff81056a1c2..0aae52cbd40 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,8 @@ Core and Builtins Library ------- +- Issue #23171: csv.Writer.writerow() now supports arbitrary iterables. + - Issue #23745: The new email header parser now handles duplicate MIME parameter names without error, similar to how get_param behaves. diff --git a/Modules/_csv.c b/Modules/_csv.c index ade35e5bf71..eb886264d7a 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1009,7 +1009,7 @@ join_reset(WriterObj *self) */ static Py_ssize_t join_append_data(WriterObj *self, unsigned int field_kind, void *field_data, - Py_ssize_t field_len, int quote_empty, int *quoted, + Py_ssize_t field_len, int *quoted, int copy_phase) { DialectObj *dialect = self->dialect; @@ -1071,18 +1071,6 @@ join_append_data(WriterObj *self, unsigned int field_kind, void *field_data, ADDCH(c); } - /* If field is empty check if it needs to be quoted. - */ - if (i == 0 && quote_empty) { - if (dialect->quoting == QUOTE_NONE) { - PyErr_Format(_csvstate_global->error_obj, - "single empty field record must be quoted"); - return -1; - } - else - *quoted = 1; - } - if (*quoted) { if (copy_phase) ADDCH(dialect->quotechar); @@ -1126,7 +1114,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) } static int -join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) +join_append(WriterObj *self, PyObject *field, int quoted) { unsigned int field_kind = -1; void *field_data = NULL; @@ -1141,7 +1129,7 @@ join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) field_len = PyUnicode_GET_LENGTH(field); } rec_len = join_append_data(self, field_kind, field_data, field_len, - quote_empty, quoted, 0); + "ed, 0); if (rec_len < 0) return 0; @@ -1150,7 +1138,7 @@ join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) return 0; self->rec_len = join_append_data(self, field_kind, field_data, field_len, - quote_empty, quoted, 1); + "ed, 1); self->num_fields++; return 1; @@ -1181,37 +1169,30 @@ join_append_lineterminator(WriterObj *self) } PyDoc_STRVAR(csv_writerow_doc, -"writerow(sequence)\n" +"writerow(iterable)\n" "\n" -"Construct and write a CSV record from a sequence of fields. Non-string\n" +"Construct and write a CSV record from an iterable of fields. Non-string\n" "elements will be converted to string."); static PyObject * csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; - Py_ssize_t len, i; - PyObject *line, *result; + PyObject *iter, *field, *line, *result; - if (!PySequence_Check(seq)) - return PyErr_Format(_csvstate_global->error_obj, "sequence expected"); - - len = PySequence_Length(seq); - if (len < 0) - return NULL; + iter = PyObject_GetIter(seq); + if (iter == NULL) + return PyErr_Format(_csvstate_global->error_obj, + "iterable expected, not %.200s", + seq->ob_type->tp_name); /* Join all fields in internal buffer. */ join_reset(self); - for (i = 0; i < len; i++) { - PyObject *field; + while ((field = PyIter_Next(iter))) { int append_ok; int quoted; - field = PySequence_GetItem(seq, i); - if (field == NULL) - return NULL; - switch (dialect->quoting) { case QUOTE_NONNUMERIC: quoted = !PyNumber_Check(field); @@ -1225,11 +1206,11 @@ csv_writerow(WriterObj *self, PyObject *seq) } if (PyUnicode_Check(field)) { - append_ok = join_append(self, field, "ed, len == 1); + append_ok = join_append(self, field, quoted); Py_DECREF(field); } else if (field == Py_None) { - append_ok = join_append(self, NULL, "ed, len == 1); + append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } else { @@ -1237,19 +1218,37 @@ csv_writerow(WriterObj *self, PyObject *seq) str = PyObject_Str(field); Py_DECREF(field); - if (str == NULL) + if (str == NULL) { + Py_DECREF(iter); return NULL; - append_ok = join_append(self, str, "ed, len == 1); + } + append_ok = join_append(self, str, quoted); Py_DECREF(str); } - if (!append_ok) + if (!append_ok) { + Py_DECREF(iter); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) + return NULL; + + if (self->num_fields > 0 && self->rec_size == 0) { + if (dialect->quoting == QUOTE_NONE) { + PyErr_Format(_csvstate_global->error_obj, + "single empty field record must be quoted"); + return NULL; + } + self->num_fields--; + if (!join_append(self, NULL, 1)) return NULL; } /* Add line terminator. */ if (!join_append_lineterminator(self)) - return 0; + return NULL; line = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, (void *) self->rec, self->rec_len); @@ -1261,9 +1260,9 @@ csv_writerow(WriterObj *self, PyObject *seq) } PyDoc_STRVAR(csv_writerows_doc, -"writerows(sequence of sequences)\n" +"writerows(iterable of iterables)\n" "\n" -"Construct and write a series of sequences to a csv file. Non-string\n" +"Construct and write a series of iterables to a csv file. Non-string\n" "elements will be converted to string."); static PyObject * From 2c7b5a9d0d6587a38f24da1f96f581bad0d3ded6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:19:08 +0300 Subject: [PATCH 16/20] Issue #23466: %c, %o, %x, and %X in bytes formatting now raise TypeError on non-integer input. --- Lib/test/test_format.py | 19 +++++++++- Misc/NEWS | 3 ++ Objects/bytesobject.c | 81 ++++++++++++++++++++++++++++------------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index e1ea33fbbde..5a2a357e081 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -272,9 +272,18 @@ def test_exc(formatstr, args, exception, excmsg): #test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError, # "unsupported format character '?' (0x3000) at index 5") test_exc('%d', '1', TypeError, "%d format: a number is required, not str") + test_exc('%x', '1', TypeError, "%x format: a number is required, not str") + test_exc('%x', 3.14, TypeError, "%x format: an integer is required, not float") test_exc('%g', '1', TypeError, "a float is required") test_exc('no format', '1', TypeError, "not all arguments converted during string formatting") + test_exc('%c', -1, OverflowError, "%c arg not in range(0x110000)") + test_exc('%c', sys.maxunicode+1, OverflowError, + "%c arg not in range(0x110000)") + #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") + test_exc('%c', 3.14, TypeError, "%c requires int or char") + test_exc('%c', 'ab', TypeError, "%c requires int or char") + test_exc('%c', b'x', TypeError, "%c requires int or char") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -339,6 +348,8 @@ def test_exc(formatstr, args, exception, excmsg): "%d format: a number is required, not str") test_exc(b'%d', b'1', TypeError, "%d format: a number is required, not bytes") + test_exc(b'%x', 3.14, TypeError, + "%x format: an integer is required, not float") test_exc(b'%g', '1', TypeError, "float argument required, not str") test_exc(b'%g', b'1', TypeError, "float argument required, not bytes") test_exc(b'no format', 7, TypeError, @@ -347,11 +358,17 @@ def test_exc(formatstr, args, exception, excmsg): "not all arguments converted during bytes formatting") test_exc(b'no format', bytearray(b'1'), TypeError, "not all arguments converted during bytes formatting") + test_exc(b"%c", -1, TypeError, + "%c requires an integer in range(256) or a single byte") test_exc(b"%c", 256, TypeError, "%c requires an integer in range(256) or a single byte") + test_exc(b"%c", 2**128, TypeError, + "%c requires an integer in range(256) or a single byte") test_exc(b"%c", b"Za", TypeError, "%c requires an integer in range(256) or a single byte") - test_exc(b"%c", "Yb", TypeError, + test_exc(b"%c", "Y", TypeError, + "%c requires an integer in range(256) or a single byte") + test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, "%b requires bytes, or an object that implements __bytes__, not 'str'") diff --git a/Misc/NEWS b/Misc/NEWS index 0aae52cbd40..36415588b75 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Release date: 2015-03-28 Core and Builtins ----------------- +- Issue #23466: %c, %o, %x, and %X in bytes formatting now raise TypeError on + non-integer input. + - Issue #23573: Increased performance of string search operations (str.find, str.index, str.count, the in operator, str.split, str.partition) with arguments of different kinds (UCS1, UCS2, UCS4). diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4d6b3e4abe1..5a2d41c5a8f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -433,7 +433,41 @@ formatfloat(PyObject *v, int flags, int prec, int type) return result; } -Py_LOCAL_INLINE(int) +static PyObject * +formatlong(PyObject *v, int flags, int prec, int type) +{ + PyObject *result, *iobj; + if (type == 'i') + type = 'd'; + if (PyLong_Check(v)) + return _PyUnicode_FormatLong(v, flags & F_ALT, prec, type); + if (PyNumber_Check(v)) { + /* make sure number is a type of integer for o, x, and X */ + if (type == 'o' || type == 'x' || type == 'X') + iobj = PyNumber_Index(v); + else + iobj = PyNumber_Long(v); + if (iobj == NULL) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return NULL; + } + else if (!PyLong_Check(iobj)) + Py_CLEAR(iobj); + if (iobj != NULL) { + result = _PyUnicode_FormatLong(iobj, flags & F_ALT, prec, type); + Py_DECREF(iobj); + return result; + } + } + PyErr_Format(PyExc_TypeError, + "%%%c format: %s is required, not %.200s", type, + (type == 'o' || type == 'x' || type == 'X') ? "an integer" + : "a number", + Py_TYPE(v)->tp_name); + return NULL; +} + +static int byte_converter(PyObject *arg, char *p) { if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) { @@ -445,12 +479,29 @@ byte_converter(PyObject *arg, char *p) return 1; } else { - long ival = PyLong_AsLong(arg); - if (0 <= ival && ival <= 255) { + PyObject *iobj; + long ival; + int overflow; + /* make sure number is a type of integer */ + if (PyLong_Check(arg)) { + ival = PyLong_AsLongAndOverflow(arg, &overflow); + } + else { + iobj = PyNumber_Index(arg); + if (iobj == NULL) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return 0; + goto onError; + } + ival = PyLong_AsLongAndOverflow(iobj, &overflow); + Py_DECREF(iobj); + } + if (!overflow && 0 <= ival && ival <= 255) { *p = (char)ival; return 1; } } + onError: PyErr_SetString(PyExc_TypeError, "%c requires an integer in range(256) or a single byte"); return 0; @@ -561,7 +612,6 @@ _PyBytes_Format(PyObject *format, PyObject *args) int prec = -1; int c = '\0'; int fill; - PyObject *iobj; PyObject *v = NULL; PyObject *temp = NULL; const char *pbuf = NULL; @@ -747,28 +797,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'o': case 'x': case 'X': - if (c == 'i') - c = 'd'; - iobj = NULL; - if (PyNumber_Check(v)) { - if ((PyLong_Check(v))) { - iobj = v; - Py_INCREF(iobj); - } - else { - iobj = PyNumber_Long(v); - if (iobj != NULL && !PyLong_Check(iobj)) - Py_CLEAR(iobj); - } - } - if (iobj == NULL) { - PyErr_Format(PyExc_TypeError, - "%%%c format: a number is required, " - "not %.200s", c, Py_TYPE(v)->tp_name); - goto error; - } - temp = _PyUnicode_FormatLong(iobj, flags & F_ALT, prec, c); - Py_DECREF(iobj); + temp = formatlong(v, flags, prec, c); if (!temp) goto error; assert(PyUnicode_IS_ASCII(temp)); From 4aa867959f19ce92ccf999aa9f9dc4bd0b49ead7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:52:29 +0300 Subject: [PATCH 17/20] Issue #23783: Fixed memory leak in PyObject_ClearWeakRefs() in case of MemoryError. --- Objects/weakrefobject.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index c083f8fce5e..d4d52e60aea 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -900,11 +900,9 @@ PyObject_ClearWeakRefs(PyObject *object) if (*list != NULL) { PyWeakReference *current = *list; Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - int restore_error = PyErr_Occurred() ? 1 : 0; PyObject *err_type, *err_value, *err_tb; - if (restore_error) - PyErr_Fetch(&err_type, &err_value, &err_tb); + PyErr_Fetch(&err_type, &err_value, &err_tb); if (count == 1) { PyObject *callback = current->wr_callback; @@ -922,8 +920,7 @@ PyObject_ClearWeakRefs(PyObject *object) tuple = PyTuple_New(count * 2); if (tuple == NULL) { - if (restore_error) - PyErr_Fetch(&err_type, &err_value, &err_tb); + _PyErr_ChainExceptions(err_type, err_value, err_tb); return; } @@ -954,7 +951,7 @@ PyObject_ClearWeakRefs(PyObject *object) } Py_DECREF(tuple); } - if (restore_error) - PyErr_Restore(err_type, err_value, err_tb); + assert(!PyErr_Occurred()); + PyErr_Restore(err_type, err_value, err_tb); } } From 04d09ebd396d2a87ad7204e170c1149edc9b141d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Mar 2015 09:58:41 +0300 Subject: [PATCH 18/20] Issue #23785: Fixed memory leak in TextIOWrapper.tell() in rare circumstances. --- Modules/_io/textio.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3f01980af15..d1c0d012328 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2435,14 +2435,10 @@ textiowrapper_tell(textio *self, PyObject *args) if (saved_state) { PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); - res = _PyObject_CallMethodId(self->decoder, &PyId_setstate, "(O)", saved_state); + _PyErr_ChainExceptions(type, value, traceback); Py_DECREF(saved_state); - if (res == NULL) - return NULL; - Py_DECREF(res); - - PyErr_Restore(type, value, traceback); + Py_XDECREF(res); } return NULL; } From e134a7fe36652434c2ccffc4ebab2ec2031d1505 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 10:09:31 +0200 Subject: [PATCH 19/20] Issue #23752: _Py_fstat() is now responsible to raise the Python exception Add _Py_fstat_noraise() function when a Python exception is not welcome. --- Include/fileutils.h | 8 +++-- Modules/_io/fileio.c | 10 +++---- Modules/main.c | 6 ++-- Modules/mmapmodule.c | 25 ++++++++-------- Modules/posixmodule.c | 2 +- Modules/signalmodule.c | 10 ++----- Programs/_freeze_importlib.c | 6 ++-- Python/dynload_shlib.c | 14 ++++----- Python/fileutils.c | 58 +++++++++++++++++++++++++++++------- Python/marshal.c | 2 +- Python/random.c | 3 +- Python/sysmodule.c | 2 +- 12 files changed, 89 insertions(+), 57 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 93a9297c87a..4fd3172dc5d 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -41,12 +41,16 @@ struct _Py_stat_struct { PyAPI_FUNC(int) _Py_fstat( int fd, - struct _Py_stat_struct *stat); + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_fstat_noraise( + int fd, + struct _Py_stat_struct *status); #endif /* Py_LIMITED_API */ PyAPI_FUNC(int) _Py_stat( PyObject *path, - struct stat *statbuf); + struct stat *status); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _Py_open( diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 595f99ee858..b56a9c3f435 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -399,10 +399,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) } self->blksize = DEFAULT_BUFFER_SIZE; - if (_Py_fstat(self->fd, &fdfstat) < 0) { - PyErr_SetFromErrno(PyExc_OSError); + if (_Py_fstat(self->fd, &fdfstat) < 0) goto error; - } #if defined(S_ISDIR) && defined(EISDIR) /* On Unix, open will succeed for directories. In Python, there should be no file objects referring to @@ -589,7 +587,7 @@ new_buffersize(fileio *self, size_t currentsize) static PyObject * fileio_readall(fileio *self) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; Py_off_t pos, end; PyObject *result; Py_ssize_t bytes_read = 0; @@ -606,8 +604,8 @@ fileio_readall(fileio *self) #else pos = lseek(self->fd, 0L, SEEK_CUR); #endif - if (_Py_fstat(self->fd, &st) == 0) - end = st.st_size; + if (_Py_fstat_noraise(self->fd, &status) == 0) + end = status.st_size; else end = (Py_off_t)-1; diff --git a/Modules/main.c b/Modules/main.c index 74e512bac00..2a9ea2882c2 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -753,9 +753,11 @@ Py_Main(int argc, wchar_t **argv) } { struct _Py_stat_struct sb; - if (_Py_fstat(fileno(fp), &sb) == 0 && + if (_Py_fstat_noraise(fileno(fp), &sb) == 0 && S_ISDIR(sb.st_mode)) { - fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename); + fprintf(stderr, + "%ls: '%ls' is a directory, cannot continue\n", + argv[0], filename); fclose(fp); return 1; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 25056a4aaa6..e2ed5f9c8d9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -465,15 +465,13 @@ mmap_size_method(mmap_object *self, #ifdef UNIX { - struct _Py_stat_struct buf; - if (-1 == _Py_fstat(self->fd, &buf)) { - PyErr_SetFromErrno(PyExc_OSError); + struct _Py_stat_struct status; + if (_Py_fstat(self->fd, &status) == -1) return NULL; - } #ifdef HAVE_LARGEFILE_SUPPORT - return PyLong_FromLongLong(buf.st_size); + return PyLong_FromLongLong(status.st_size); #else - return PyLong_FromLong(buf.st_size); + return PyLong_FromLong(status.st_size); #endif } #endif /* UNIX */ @@ -1112,7 +1110,7 @@ _GetMapSize(PyObject *o, const char* param) static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; mmap_object *m_obj; PyObject *map_size_obj = NULL; Py_ssize_t map_size; @@ -1177,25 +1175,26 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fd != -1) (void)fcntl(fd, F_FULLFSYNC); #endif - if (fd != -1 && _Py_fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) { + if (fd != -1 && _Py_fstat_noraise(fd, &status) == 0 + && S_ISREG(status.st_mode)) { if (map_size == 0) { - if (st.st_size == 0) { + if (status.st_size == 0) { PyErr_SetString(PyExc_ValueError, "cannot mmap an empty file"); return NULL; } - if (offset >= st.st_size) { + if (offset >= status.st_size) { PyErr_SetString(PyExc_ValueError, "mmap offset is greater than file size"); return NULL; } - if (st.st_size - offset > PY_SSIZE_T_MAX) { + if (status.st_size - offset > PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_ValueError, "mmap length is too large"); return NULL; } - map_size = (Py_ssize_t) (st.st_size - offset); - } else if (offset + map_size > st.st_size) { + map_size = (Py_ssize_t) (status.st_size - offset); + } else if (offset + map_size > status.st_size) { PyErr_SetString(PyExc_ValueError, "mmap length is greater than file size"); return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 801305f535f..ef69a45ee1b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -351,7 +351,7 @@ static int win32_can_symlink = 0; #ifdef MS_WINDOWS # define STAT win32_stat # define LSTAT win32_lstat -# define FSTAT _Py_fstat +# define FSTAT _Py_fstat_noraise # define STRUCT_STAT struct _Py_stat_struct #else # define STAT stat diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 1b3589d7c65..3081562abce 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -503,7 +503,7 @@ signal_siginterrupt(PyObject *self, PyObject *args) static PyObject * signal_set_wakeup_fd(PyObject *self, PyObject *args) { - struct _Py_stat_struct st; + struct _Py_stat_struct status; #ifdef MS_WINDOWS PyObject *fdobj; SOCKET_T sockfd, old_sockfd; @@ -559,10 +559,8 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args) return NULL; } - if (_Py_fstat(fd, &st) != 0) { - PyErr_SetExcFromWindowsErr(PyExc_OSError, GetLastError()); + if (_Py_fstat(fd, &status) != 0) return NULL; - } /* on Windows, a file cannot be set to non-blocking mode */ } @@ -591,10 +589,8 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args) return NULL; } - if (_Py_fstat(fd, &st) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + if (_Py_fstat(fd, &status) != 0) return NULL; - } blocking = _Py_get_blocking(fd); if (blocking < 0) diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 31b3d31bd7d..b72fcf486fc 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -35,7 +35,7 @@ main(int argc, char *argv[]) { char *inpath, *outpath; FILE *infile = NULL, *outfile = NULL; - struct _Py_stat_struct st; + struct _Py_stat_struct status; size_t text_size, data_size, n; char *text = NULL; unsigned char *data; @@ -54,11 +54,11 @@ main(int argc, char *argv[]) fprintf(stderr, "cannot open '%s' for reading\n", inpath); goto error; } - if (_Py_fstat(fileno(infile), &st)) { + if (_Py_fstat_noraise(fileno(infile), &status)) { fprintf(stderr, "cannot fstat '%s'\n", inpath); goto error; } - text_size = st.st_size; + text_size = status.st_size; text = (char *) malloc(text_size + 1); if (text == NULL) { fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 659adace095..1a467fde7c8 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -71,22 +71,20 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname, if (fp != NULL) { int i; - struct _Py_stat_struct statb; - if (_Py_fstat(fileno(fp), &statb) == -1) { - PyErr_SetFromErrno(PyExc_IOError); + struct _Py_stat_struct status; + if (_Py_fstat(fileno(fp), &status) == -1) return NULL; - } for (i = 0; i < nhandles; i++) { - if (statb.st_dev == handles[i].dev && - statb.st_ino == handles[i].ino) { + if (status.st_dev == handles[i].dev && + status.st_ino == handles[i].ino) { p = (dl_funcptr) dlsym(handles[i].handle, funcname); return p; } } if (nhandles < 128) { - handles[nhandles].dev = statb.st_dev; - handles[nhandles].ino = statb.st_ino; + handles[nhandles].dev = status.st_dev; + handles[nhandles].ino = status.st_ino; } } diff --git a/Python/fileutils.c b/Python/fileutils.c index e6d3154490f..daaad2a3d58 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -565,7 +565,8 @@ attributes_to_mode(DWORD attr) } void -_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, struct _Py_stat_struct *result) +_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + struct _Py_stat_struct *result) { memset(result, 0, sizeof(*result)); result->st_mode = attributes_to_mode(info->dwFileAttributes); @@ -595,9 +596,12 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger than 2 GB because the file size type is an signed 32-bit integer: see issue #23152. - */ + + On Windows, set the last Windows error and return nonzero on error. On + POSIX, set errno and return nonzero on error. Fill status and return 0 on + success. */ int -_Py_fstat(int fd, struct _Py_stat_struct *result) +_Py_fstat_noraise(int fd, struct _Py_stat_struct *status) { #ifdef MS_WINDOWS BY_HANDLE_FILE_INFORMATION info; @@ -619,22 +623,21 @@ _Py_fstat(int fd, struct _Py_stat_struct *result) SetLastError(ERROR_INVALID_HANDLE); return -1; } - memset(result, 0, sizeof(*result)); + memset(status, 0, sizeof(*status)); type = GetFileType(h); if (type == FILE_TYPE_UNKNOWN) { DWORD error = GetLastError(); - if (error != 0) { + if (error != 0) return -1; - } /* else: valid but unknown file */ } if (type != FILE_TYPE_DISK) { if (type == FILE_TYPE_CHAR) - result->st_mode = _S_IFCHR; + status->st_mode = _S_IFCHR; else if (type == FILE_TYPE_PIPE) - result->st_mode = _S_IFIFO; + status->st_mode = _S_IFIFO; return 0; } @@ -642,15 +645,48 @@ _Py_fstat(int fd, struct _Py_stat_struct *result) return -1; } - _Py_attribute_data_to_stat(&info, 0, result); + _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - result->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; return 0; #else - return fstat(fd, result); + return fstat(fd, status); #endif } +/* Return information about a file. + + On POSIX, use fstat(). + + On Windows, use GetFileType() and GetFileInformationByHandle() which support + files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger + than 2 GB because the file size type is an signed 32-bit integer: see issue + #23152. + + Raise an exception and return -1 on error. On Windows, set the last Windows + error on error. On POSIX, set errno on error. Fill status and return 0 on + success. + + The GIL must be held. */ +int +_Py_fstat(int fd, struct _Py_stat_struct *status) +{ + int res; + + Py_BEGIN_ALLOW_THREADS + res = _Py_fstat_noraise(fd, status); + Py_END_ALLOW_THREADS + + if (res != 0) { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErr(0); +#else + PyErr_SetFromErrno(PyExc_OSError); +#endif + return -1; + } + return 0; +} /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. diff --git a/Python/marshal.c b/Python/marshal.c index 3472882fb56..f89cd04b655 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1486,7 +1486,7 @@ static off_t getfilesize(FILE *fp) { struct _Py_stat_struct st; - if (_Py_fstat(fileno(fp), &st) != 0) + if (_Py_fstat_noraise(fileno(fp), &st) != 0) return -1; #if SIZEOF_OFF_T == 4 else if (st.st_size >= INT_MAX) diff --git a/Python/random.c b/Python/random.c index a281829f974..a4eba3ccd93 100644 --- a/Python/random.c +++ b/Python/random.c @@ -221,7 +221,7 @@ dev_urandom_python(char *buffer, Py_ssize_t size) if (urandom_cache.fd >= 0) { /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat(urandom_cache.fd, &st) + if (_Py_fstat_noraise(urandom_cache.fd, &st) || st.st_dev != urandom_cache.st_dev || st.st_ino != urandom_cache.st_ino) { /* Something changed: forget the cached fd (but don't close it, @@ -250,7 +250,6 @@ dev_urandom_python(char *buffer, Py_ssize_t size) } else { if (_Py_fstat(fd, &st)) { - PyErr_SetFromErrno(PyExc_OSError); close(fd); return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 471389cf577..9ec25216ed8 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1690,7 +1690,7 @@ _PySys_Init(void) #if !defined(MS_WINDOWS) { struct _Py_stat_struct sb; - if (_Py_fstat(fileno(stdin), &sb) == 0 && + if (_Py_fstat_noraise(fileno(stdin), &sb) == 0 && S_ISDIR(sb.st_mode)) { /* There's nothing more we can do. */ /* Py_FatalError() will core dump, so just exit. */ From 45cff0c0e6c4a31ed3b5b88ee803320862fbd43a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 Mar 2015 10:22:16 +0200 Subject: [PATCH 20/20] Issue #22117: Try to fix rounding in conversion from Python double to _PyTime_t using the C volatile keyword. --- Python/pytime.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/pytime.c b/Python/pytime.c index 98f29acc38b..5bf8c568e7b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -207,7 +207,8 @@ int _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) { if (PyFloat_Check(obj)) { - double d, err; + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double d, err; /* convert to a number of nanoseconds */ d = PyFloat_AsDouble(obj);