From 4932e14542e10191b6b598307641f6b54c5da239 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Fri, 20 Jun 2014 01:37:53 -0700 Subject: [PATCH 1/5] issue 20091 - index entry for __main__ in runpy docs. --- Doc/library/runpy.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index ee9fbcf529c..a764696a38a 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -28,6 +28,9 @@ The :mod:`runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) + .. index:: + module: __main__ + Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using the standard import mechanism (refer to :pep:`302` for details) and then executed in a @@ -87,6 +90,9 @@ The :mod:`runpy` module provides two functions: .. function:: run_path(file_path, init_globals=None, run_name=None) + .. index:: + module: __main__ + Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a From 0e6f52a2114d4fb3bf196e6a17e3b5daea3c0cdc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Jun 2014 17:34:15 +0200 Subject: [PATCH 2/5] asyncio, Tulip issue 105: in debug mode, log callbacks taking more than 100 ms to be executed. --- Lib/asyncio/base_events.py | 32 +++++++++++++++++++---- Lib/test/test_asyncio/test_base_events.py | 28 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 935098924dc..9f9067ed997 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,6 +17,7 @@ import collections import concurrent.futures import heapq +import inspect import logging import socket import subprocess @@ -37,6 +38,15 @@ _MAX_WORKERS = 5 +def _format_handle(handle): + cb = handle._callback + if inspect.ismethod(cb) and isinstance(cb.__self__, tasks.Task): + # format the task + return repr(cb.__self__) + else: + return str(handle) + + class _StopError(BaseException): """Raised to stop the event loop.""" @@ -128,6 +138,9 @@ def __init__(self): self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None self._debug = False + # In debug mode, if the execution of a callback or a step of a task + # exceed this duration in seconds, the slow callback/task is logged. + self.slow_callback_duration = 0.1 def __repr__(self): return ('<%s running=%s closed=%s debug=%s>' @@ -823,16 +836,16 @@ def _run_once(self): if logger.isEnabledFor(logging.INFO): t0 = self.time() event_list = self._selector.select(timeout) - t1 = self.time() - if t1-t0 >= 1: + dt = self.time() - t0 + if dt >= 1: level = logging.INFO else: level = logging.DEBUG if timeout is not None: logger.log(level, 'poll %.3f took %.3f seconds', - timeout, t1-t0) + timeout, dt) else: - logger.log(level, 'poll took %.3f seconds', t1-t0) + logger.log(level, 'poll took %.3f seconds', dt) else: event_list = self._selector.select(timeout) self._process_events(event_list) @@ -855,7 +868,16 @@ def _run_once(self): ntodo = len(self._ready) for i in range(ntodo): handle = self._ready.popleft() - if not handle._cancelled: + if handle._cancelled: + continue + if self._debug: + t0 = self.time() + handle._run() + dt = self.time() - t0 + if dt >= self.slow_callback_duration: + logger.warning('Executing %s took %.3f seconds', + _format_handle(handle), dt) + else: handle._run() handle = None # Needed to break cycles when an exception occurs. diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 059b41c329c..352af4887c0 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -969,6 +969,34 @@ def coroutine_function(): with self.assertRaises(TypeError): self.loop.run_in_executor(None, coroutine_function) + @mock.patch('asyncio.base_events.logger') + def test_log_slow_callbacks(self, m_logger): + def stop_loop_cb(loop): + loop.stop() + + @asyncio.coroutine + def stop_loop_coro(loop): + yield from () + loop.stop() + + asyncio.set_event_loop(self.loop) + self.loop.set_debug(True) + self.loop.slow_callback_duration = 0.0 + + # slow callback + self.loop.call_soon(stop_loop_cb, self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing Handle.*stop_loop_cb.* took .* seconds$") + + # slow task + asyncio.async(stop_loop_coro(self.loop), loop=self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing Task.*stop_loop_coro.* took .* seconds$") + if __name__ == '__main__': unittest.main() From fe928de955ec5043dd7c8b9388dbb0ccfabbdc80 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 20 Jun 2014 14:59:11 -0400 Subject: [PATCH 3/5] Issue #21768: fix type in test_pydoc, patch by Claudiu Popa. --- Lib/test/test_pydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 542b433cb6e..8d85c140717 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -746,7 +746,7 @@ def test_builtin(self): try: pydoc.render_doc(name) except ImportError: - self.fail('finding the doc of {!r} failed'.format(o)) + self.fail('finding the doc of {!r} failed'.format(name)) for name in ('notbuiltins', 'strrr', 'strr.translate', 'str.trrrranslate', 'builtins.strrr', From af9eb9628d833c8096e4438a48655ed07996137a Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 20 Jun 2014 15:16:35 -0400 Subject: [PATCH 4/5] Issue #21768: fix NameError in test_pydescr. Patch by Claudiu Popa. --- Lib/test/test_descr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 8bb7d6a4742..26c273a4ccc 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1149,7 +1149,7 @@ class C(object): except (TypeError, UnicodeEncodeError): pass else: - raise TestFailed("[chr(128)] slots not caught") + self.fail("[chr(128)] slots not caught") # Test leaks class Counted(object): From 504f5c36ef2419cb0c56450c3a733499c4576456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Fri, 20 Jun 2014 22:37:35 +0100 Subject: [PATCH 5/5] Issue #21491: socketserver: Fix a race condition in child processes reaping. --- Lib/socketserver.py | 58 ++++++++++++++++++++++++--------------------- Misc/NEWS | 2 ++ 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 46ee7c54ef9..2f395fac242 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -523,35 +523,39 @@ class ForkingMixIn: def collect_children(self): """Internal routine to wait for children that have exited.""" - if self.active_children is None: return - while len(self.active_children) >= self.max_children: - # XXX: This will wait for any child process, not just ones - # spawned by this library. This could confuse other - # libraries that expect to be able to wait for their own - # children. - try: - pid, status = os.waitpid(0, 0) - except OSError: - pid = None - if pid not in self.active_children: continue - self.active_children.remove(pid) + if self.active_children is None: + return - # XXX: This loop runs more system calls than it ought - # to. There should be a way to put the active_children into a - # process group and then use os.waitpid(-pgid) to wait for any - # of that set, but I couldn't find a way to allocate pgids - # that couldn't collide. - for child in self.active_children: + # If we're above the max number of children, wait and reap them until + # we go back below threshold. Note that we use waitpid(-1) below to be + # able to collect children in size() syscalls instead + # of size(): the downside is that this might reap children + # which we didn't spawn, which is why we only resort to this when we're + # above max_children. + while len(self.active_children) >= self.max_children: try: - pid, status = os.waitpid(child, os.WNOHANG) + pid, _ = os.waitpid(-1, 0) + self.active_children.discard(pid) + except InterruptedError: + pass + except ChildProcessError: + # we don't have any children, we're done + self.active_children.clear() except OSError: - pid = None - if not pid: continue + break + + # Now reap all defunct children. + for pid in self.active_children.copy(): try: - self.active_children.remove(pid) - except ValueError as e: - raise ValueError('%s. x=%d and list=%r' % (e.message, pid, - self.active_children)) + pid, _ = os.waitpid(pid, os.WNOHANG) + # if the child hasn't exited yet, pid will be 0 and ignored by + # discard() below + self.active_children.discard(pid) + except ChildProcessError: + # someone else reaped it + self.active_children.discard(pid) + except OSError: + pass def handle_timeout(self): """Wait for zombies after self.timeout seconds of inactivity. @@ -573,8 +577,8 @@ def process_request(self, request, client_address): if pid: # Parent process if self.active_children is None: - self.active_children = [] - self.active_children.append(pid) + self.active_children = set() + self.active_children.add(pid) self.close_request(request) return else: diff --git a/Misc/NEWS b/Misc/NEWS index e6c94694df6..b8e7219197b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,8 @@ Core and Builtins Library ------- +- Issue #21491: socketserver: Fix a race condition in child processes reaping. + - Issue #21722: The distutils "upload" command now exits with a non-zero return code when uploading fails. Patch by Martin Dengler.