bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921) (GH-28014)

Fixes:
Traceback (most recent call last):
  File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in <module>
    sys.exit(main())
  File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main
    test_all_tasks_threading()
  File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading
    results.append(f.result())
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result
    return self.__get_result()
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result
    raise self._exception
  File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run
    _cancel_all_tasks(loop)
  File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks
    to_cancel = tasks.all_tasks(loop)
  File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks
    tasks = list(_all_tasks)
  File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__
    with _IterationGuard(self):
  File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__
    w._commit_removals()
  File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals
    discard(l.pop())
IndexError: pop from empty list

Also fixes:
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
  File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
    del self.data[k]
KeyError: <weakref at 0x00007fe76e8d8180; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
  File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
    del self.data[k]
KeyError: <weakref at 0x00007fe76e8d81a0; dead>
Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0>
Traceback (most recent call last):
  File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove
    del self.data[k]
KeyError: <weakref at 0x000056548f1e24a0; dead>

See: https://github.com/agronholm/anyio/issues/362GH-issuecomment-904424310
See also: https://bugs.python.org/issue29519

Co-authored-by: Łukasz Langa <lukasz@langa.pl>
(cherry picked from commit 206b21ed9f)

Co-authored-by: Thomas Grainger <tagrain@gmail.com>

Lib/_weakrefset.py

@@ -51,10 +51,14 @@ def _remove(item, selfref=ref(self)):
             self.update(data)
 
     def _commit_removals(self):
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         discard = self.data.discard
-        while l:
-            discard(l.pop())
+        while True:
+            try:
+                item = pop()
+            except IndexError:
+                return
+            discard(item)
 
     def __iter__(self):
         with _IterationGuard(self):

Lib/weakref.py

@@ -119,14 +119,17 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
         self.data = {}
         self.update(other, **kw)
 
-    def _commit_removals(self):
-        l = self._pending_removals
+    def _commit_removals(self, _atomic_removal=_remove_dead_weakref):
+        pop = self._pending_removals.pop
         d = self.data
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
-        while l:
-            key = l.pop()
-            _remove_dead_weakref(d, key)
+        while True:
+            try:
+                key = pop()
+            except IndexError:
+                return
+            _atomic_removal(d, key)
 
     def __getitem__(self, key):
         if self._pending_removals:
@@ -370,7 +373,10 @@ def remove(k, selfref=ref(self)):
                 if self._iterating:
                     self._pending_removals.append(k)
                 else:
+                    try:
                         del self.data[k]
+                    except KeyError:
+                        pass
         self._remove = remove
         # A list of dead weakrefs (keys to be removed)
         self._pending_removals = []
@@ -384,11 +390,16 @@ def _commit_removals(self):
         # because a dead weakref never compares equal to a live weakref,
         # even if they happened to refer to equal objects.
         # However, it means keys may already have been removed.
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         d = self.data
-        while l:
+        while True:
             try:
-                del d[l.pop()]
+                key = pop()
+            except IndexError:
+                return
+
+            try:
+                del d[key]
             except KeyError:
                 pass
 

Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst

@@ -0,0 +1 @@
+Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run