mirror of
https://github.com/python/cpython.git
synced 2025-11-03 23:21:29 +00:00
ead8b7ab30
In cyclic gc, clear weakrefs to unreachable objects before allowing any Python code (weakref callbacks or __del__ methods) to run. This is a critical bugfix, affecting all versions of Python since weakrefs were introduced. I'll backport to 2.3.
618 lines
17 KiB
Python
618 lines
17 KiB
Python
from test.test_support import verify, verbose, TestFailed, vereq
|
|
import sys
|
|
import gc
|
|
import weakref
|
|
|
|
def expect(actual, expected, name):
|
|
if actual != expected:
|
|
raise TestFailed, "test_%s: actual %r, expected %r" % (
|
|
name, actual, expected)
|
|
|
|
def expect_nonzero(actual, name):
|
|
if actual == 0:
|
|
raise TestFailed, "test_%s: unexpected zero" % name
|
|
|
|
def run_test(name, thunk):
|
|
if verbose:
|
|
print "testing %s..." % name,
|
|
thunk()
|
|
if verbose:
|
|
print "ok"
|
|
|
|
def test_list():
|
|
l = []
|
|
l.append(l)
|
|
gc.collect()
|
|
del l
|
|
expect(gc.collect(), 1, "list")
|
|
|
|
def test_dict():
|
|
d = {}
|
|
d[1] = d
|
|
gc.collect()
|
|
del d
|
|
expect(gc.collect(), 1, "dict")
|
|
|
|
def test_tuple():
|
|
# since tuples are immutable we close the loop with a list
|
|
l = []
|
|
t = (l,)
|
|
l.append(t)
|
|
gc.collect()
|
|
del t
|
|
del l
|
|
expect(gc.collect(), 2, "tuple")
|
|
|
|
def test_class():
|
|
class A:
|
|
pass
|
|
A.a = A
|
|
gc.collect()
|
|
del A
|
|
expect_nonzero(gc.collect(), "class")
|
|
|
|
def test_newstyleclass():
|
|
class A(object):
|
|
pass
|
|
gc.collect()
|
|
del A
|
|
expect_nonzero(gc.collect(), "staticclass")
|
|
|
|
def test_instance():
|
|
class A:
|
|
pass
|
|
a = A()
|
|
a.a = a
|
|
gc.collect()
|
|
del a
|
|
expect_nonzero(gc.collect(), "instance")
|
|
|
|
def test_newinstance():
|
|
class A(object):
|
|
pass
|
|
a = A()
|
|
a.a = a
|
|
gc.collect()
|
|
del a
|
|
expect_nonzero(gc.collect(), "newinstance")
|
|
class B(list):
|
|
pass
|
|
class C(B, A):
|
|
pass
|
|
a = C()
|
|
a.a = a
|
|
gc.collect()
|
|
del a
|
|
expect_nonzero(gc.collect(), "newinstance(2)")
|
|
del B, C
|
|
expect_nonzero(gc.collect(), "newinstance(3)")
|
|
A.a = A()
|
|
del A
|
|
expect_nonzero(gc.collect(), "newinstance(4)")
|
|
expect(gc.collect(), 0, "newinstance(5)")
|
|
|
|
def test_method():
|
|
# Tricky: self.__init__ is a bound method, it references the instance.
|
|
class A:
|
|
def __init__(self):
|
|
self.init = self.__init__
|
|
a = A()
|
|
gc.collect()
|
|
del a
|
|
expect_nonzero(gc.collect(), "method")
|
|
|
|
def test_finalizer():
|
|
# A() is uncollectable if it is part of a cycle, make sure it shows up
|
|
# in gc.garbage.
|
|
class A:
|
|
def __del__(self): pass
|
|
class B:
|
|
pass
|
|
a = A()
|
|
a.a = a
|
|
id_a = id(a)
|
|
b = B()
|
|
b.b = b
|
|
gc.collect()
|
|
del a
|
|
del b
|
|
expect_nonzero(gc.collect(), "finalizer")
|
|
for obj in gc.garbage:
|
|
if id(obj) == id_a:
|
|
del obj.a
|
|
break
|
|
else:
|
|
raise TestFailed, "didn't find obj in garbage (finalizer)"
|
|
gc.garbage.remove(obj)
|
|
|
|
def test_finalizer_newclass():
|
|
# A() is uncollectable if it is part of a cycle, make sure it shows up
|
|
# in gc.garbage.
|
|
class A(object):
|
|
def __del__(self): pass
|
|
class B(object):
|
|
pass
|
|
a = A()
|
|
a.a = a
|
|
id_a = id(a)
|
|
b = B()
|
|
b.b = b
|
|
gc.collect()
|
|
del a
|
|
del b
|
|
expect_nonzero(gc.collect(), "finalizer")
|
|
for obj in gc.garbage:
|
|
if id(obj) == id_a:
|
|
del obj.a
|
|
break
|
|
else:
|
|
raise TestFailed, "didn't find obj in garbage (finalizer)"
|
|
gc.garbage.remove(obj)
|
|
|
|
def test_function():
|
|
# Tricky: f -> d -> f, code should call d.clear() after the exec to
|
|
# break the cycle.
|
|
d = {}
|
|
exec("def f(): pass\n") in d
|
|
gc.collect()
|
|
del d
|
|
expect(gc.collect(), 2, "function")
|
|
|
|
def test_frame():
|
|
def f():
|
|
frame = sys._getframe()
|
|
gc.collect()
|
|
f()
|
|
expect(gc.collect(), 1, "frame")
|
|
|
|
|
|
def test_saveall():
|
|
# Verify that cyclic garbage like lists show up in gc.garbage if the
|
|
# SAVEALL option is enabled.
|
|
|
|
# First make sure we don't save away other stuff that just happens to
|
|
# be waiting for collection.
|
|
gc.collect()
|
|
vereq(gc.garbage, []) # if this fails, someone else created immortal trash
|
|
|
|
L = []
|
|
L.append(L)
|
|
id_L = id(L)
|
|
|
|
debug = gc.get_debug()
|
|
gc.set_debug(debug | gc.DEBUG_SAVEALL)
|
|
del L
|
|
gc.collect()
|
|
gc.set_debug(debug)
|
|
|
|
vereq(len(gc.garbage), 1)
|
|
obj = gc.garbage.pop()
|
|
vereq(id(obj), id_L)
|
|
|
|
def test_del():
|
|
# __del__ methods can trigger collection, make this to happen
|
|
thresholds = gc.get_threshold()
|
|
gc.enable()
|
|
gc.set_threshold(1)
|
|
|
|
class A:
|
|
def __del__(self):
|
|
dir(self)
|
|
a = A()
|
|
del a
|
|
|
|
gc.disable()
|
|
gc.set_threshold(*thresholds)
|
|
|
|
def test_del_newclass():
|
|
# __del__ methods can trigger collection, make this to happen
|
|
thresholds = gc.get_threshold()
|
|
gc.enable()
|
|
gc.set_threshold(1)
|
|
|
|
class A(object):
|
|
def __del__(self):
|
|
dir(self)
|
|
a = A()
|
|
del a
|
|
|
|
gc.disable()
|
|
gc.set_threshold(*thresholds)
|
|
|
|
class Ouch:
|
|
n = 0
|
|
def __del__(self):
|
|
Ouch.n = Ouch.n + 1
|
|
if Ouch.n % 17 == 0:
|
|
gc.collect()
|
|
|
|
def test_trashcan():
|
|
# "trashcan" is a hack to prevent stack overflow when deallocating
|
|
# very deeply nested tuples etc. It works in part by abusing the
|
|
# type pointer and refcount fields, and that can yield horrible
|
|
# problems when gc tries to traverse the structures.
|
|
# If this test fails (as it does in 2.0, 2.1 and 2.2), it will
|
|
# most likely die via segfault.
|
|
|
|
# Note: In 2.3 the possibility for compiling without cyclic gc was
|
|
# removed, and that in turn allows the trashcan mechanism to work
|
|
# via much simpler means (e.g., it never abuses the type pointer or
|
|
# refcount fields anymore). Since it's much less likely to cause a
|
|
# problem now, the various constants in this expensive (we force a lot
|
|
# of full collections) test are cut back from the 2.2 version.
|
|
gc.enable()
|
|
N = 150
|
|
for count in range(2):
|
|
t = []
|
|
for i in range(N):
|
|
t = [t, Ouch()]
|
|
u = []
|
|
for i in range(N):
|
|
u = [u, Ouch()]
|
|
v = {}
|
|
for i in range(N):
|
|
v = {1: v, 2: Ouch()}
|
|
gc.disable()
|
|
|
|
class Boom:
|
|
def __getattr__(self, someattribute):
|
|
del self.attr
|
|
raise AttributeError
|
|
|
|
def test_boom():
|
|
a = Boom()
|
|
b = Boom()
|
|
a.attr = b
|
|
b.attr = a
|
|
|
|
gc.collect()
|
|
garbagelen = len(gc.garbage)
|
|
del a, b
|
|
# a<->b are in a trash cycle now. Collection will invoke Boom.__getattr__
|
|
# (to see whether a and b have __del__ methods), and __getattr__ deletes
|
|
# the internal "attr" attributes as a side effect. That causes the
|
|
# trash cycle to get reclaimed via refcounts falling to 0, thus mutating
|
|
# the trash graph as a side effect of merely asking whether __del__
|
|
# exists. This used to (before 2.3b1) crash Python. Now __getattr__
|
|
# isn't called.
|
|
expect(gc.collect(), 4, "boom")
|
|
expect(len(gc.garbage), garbagelen, "boom")
|
|
|
|
class Boom2:
|
|
def __init__(self):
|
|
self.x = 0
|
|
|
|
def __getattr__(self, someattribute):
|
|
self.x += 1
|
|
if self.x > 1:
|
|
del self.attr
|
|
raise AttributeError
|
|
|
|
def test_boom2():
|
|
a = Boom2()
|
|
b = Boom2()
|
|
a.attr = b
|
|
b.attr = a
|
|
|
|
gc.collect()
|
|
garbagelen = len(gc.garbage)
|
|
del a, b
|
|
# Much like test_boom(), except that __getattr__ doesn't break the
|
|
# cycle until the second time gc checks for __del__. As of 2.3b1,
|
|
# there isn't a second time, so this simply cleans up the trash cycle.
|
|
# We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get reclaimed
|
|
# this way.
|
|
expect(gc.collect(), 4, "boom2")
|
|
expect(len(gc.garbage), garbagelen, "boom2")
|
|
|
|
# boom__new and boom2_new are exactly like boom and boom2, except use
|
|
# new-style classes.
|
|
|
|
class Boom_New(object):
|
|
def __getattr__(self, someattribute):
|
|
del self.attr
|
|
raise AttributeError
|
|
|
|
def test_boom_new():
|
|
a = Boom_New()
|
|
b = Boom_New()
|
|
a.attr = b
|
|
b.attr = a
|
|
|
|
gc.collect()
|
|
garbagelen = len(gc.garbage)
|
|
del a, b
|
|
expect(gc.collect(), 4, "boom_new")
|
|
expect(len(gc.garbage), garbagelen, "boom_new")
|
|
|
|
class Boom2_New(object):
|
|
def __init__(self):
|
|
self.x = 0
|
|
|
|
def __getattr__(self, someattribute):
|
|
self.x += 1
|
|
if self.x > 1:
|
|
del self.attr
|
|
raise AttributeError
|
|
|
|
def test_boom2_new():
|
|
a = Boom2_New()
|
|
b = Boom2_New()
|
|
a.attr = b
|
|
b.attr = a
|
|
|
|
gc.collect()
|
|
garbagelen = len(gc.garbage)
|
|
del a, b
|
|
expect(gc.collect(), 4, "boom2_new")
|
|
expect(len(gc.garbage), garbagelen, "boom2_new")
|
|
|
|
def test_get_referents():
|
|
alist = [1, 3, 5]
|
|
got = gc.get_referents(alist)
|
|
got.sort()
|
|
expect(got, alist, "get_referents")
|
|
|
|
atuple = tuple(alist)
|
|
got = gc.get_referents(atuple)
|
|
got.sort()
|
|
expect(got, alist, "get_referents")
|
|
|
|
adict = {1: 3, 5: 7}
|
|
expected = [1, 3, 5, 7]
|
|
got = gc.get_referents(adict)
|
|
got.sort()
|
|
expect(got, expected, "get_referents")
|
|
|
|
got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
|
|
got.sort()
|
|
expect(got, [0, 0] + range(5), "get_referents")
|
|
|
|
expect(gc.get_referents(1, 'a', 4j), [], "get_referents")
|
|
|
|
# Bug 1055820 has several tests of longstanding bugs involving weakrefs and
|
|
# cyclic gc.
|
|
|
|
# An instance of C1055820 has a self-loop, so becomes cyclic trash when
|
|
# unreachable.
|
|
class C1055820(object):
|
|
def __init__(self, i):
|
|
self.i = i
|
|
self.loop = self
|
|
|
|
class GC_Detector(object):
|
|
# Create an instance I. Then gc hasn't happened again so long as
|
|
# I.gc_happened is false.
|
|
|
|
def __init__(self):
|
|
self.gc_happened = False
|
|
|
|
def it_happened(ignored):
|
|
self.gc_happened = True
|
|
|
|
# Create a piece of cyclic trash that triggers it_happened when
|
|
# gc collects it.
|
|
self.wr = weakref.ref(C1055820(666), it_happened)
|
|
|
|
def test_bug1055820b():
|
|
# Corresponds to temp2b.py in the bug report.
|
|
|
|
ouch = []
|
|
def callback(ignored):
|
|
ouch[:] = [wr() for wr in WRs]
|
|
|
|
Cs = [C1055820(i) for i in range(2)]
|
|
WRs = [weakref.ref(c, callback) for c in Cs]
|
|
c = None
|
|
|
|
gc.collect()
|
|
expect(len(ouch), 0, "bug1055820b")
|
|
# Make the two instances trash, and collect again. The bug was that
|
|
# the callback materialized a strong reference to an instance, but gc
|
|
# cleared the instance's dict anyway.
|
|
Cs = None
|
|
gc.collect()
|
|
expect(len(ouch), 2, "bug1055820b") # else the callbacks didn't run
|
|
for x in ouch:
|
|
# If the callback resurrected one of these guys, the instance
|
|
# would be damaged, with an empty __dict__.
|
|
expect(x, None, "bug1055820b")
|
|
|
|
def test_bug1055820c():
|
|
# Corresponds to temp2c.py in the bug report. This is pretty elaborate.
|
|
|
|
c0 = C1055820(0)
|
|
# Move c0 into generation 2.
|
|
gc.collect()
|
|
|
|
c1 = C1055820(1)
|
|
c1.keep_c0_alive = c0
|
|
del c0.loop # now only c1 keeps c0 alive
|
|
|
|
c2 = C1055820(2)
|
|
c2wr = weakref.ref(c2) # no callback!
|
|
|
|
ouch = []
|
|
def callback(ignored):
|
|
ouch[:] = [c2wr()]
|
|
|
|
# The callback gets associated with a wr on an object in generation 2.
|
|
c0wr = weakref.ref(c0, callback)
|
|
|
|
c0 = c1 = c2 = None
|
|
|
|
# What we've set up: c0, c1, and c2 are all trash now. c0 is in
|
|
# generation 2. The only thing keeping it alive is that c1 points to it.
|
|
# c1 and c2 are in generation 0, and are in self-loops. There's a global
|
|
# weakref to c2 (c2wr), but that weakref has no callback. There's also
|
|
# a global weakref to c0 (c0wr), and that does have a callback, and that
|
|
# callback references c2 via c2wr().
|
|
#
|
|
# c0 has a wr with callback, which references c2wr
|
|
# ^
|
|
# |
|
|
# | Generation 2 above dots
|
|
#. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
|
|
# | Generation 0 below dots
|
|
# |
|
|
# |
|
|
# ^->c1 ^->c2 has a wr but no callback
|
|
# | | | |
|
|
# <--v <--v
|
|
#
|
|
# So this is the nightmare: when generation 0 gets collected, we see that
|
|
# c2 has a callback-free weakref, and c1 doesn't even have a weakref.
|
|
# Collecting generation 0 doesn't see c0 at all, and c0 is the only object
|
|
# that has a weakref with a callback. gc clears c1 and c2. Clearing c1
|
|
# has the side effect of dropping the refcount on c0 to 0, so c0 goes
|
|
# away (despite that it's in an older generation) and c0's wr callback
|
|
# triggers. That in turn materializes a reference to c2 via c2wr(), but
|
|
# c2 gets cleared anyway by gc.
|
|
|
|
# We want to let gc happen "naturally", to preserve the distinction
|
|
# between generations.
|
|
junk = []
|
|
i = 0
|
|
detector = GC_Detector()
|
|
while not detector.gc_happened:
|
|
i += 1
|
|
if i > 10000:
|
|
raise TestFailed("gc didn't happen after 10000 iterations")
|
|
expect(len(ouch), 0, "bug1055820c")
|
|
junk.append([]) # this will eventually trigger gc
|
|
|
|
expect(len(ouch), 1, "bug1055820c") # else the callback wasn't invoked
|
|
for x in ouch:
|
|
# If the callback resurrected c2, the instance would be damaged,
|
|
# with an empty __dict__.
|
|
expect(x, None, "bug1055820c")
|
|
|
|
def test_bug1055820d():
|
|
# Corresponds to temp2d.py in the bug report. This is very much like
|
|
# test_bug1055820c, but uses a __del__ method instead of a weakref
|
|
# callback to sneak in a resurrection of cyclic trash.
|
|
|
|
ouch = []
|
|
class D(C1055820):
|
|
def __del__(self):
|
|
ouch[:] = [c2wr()]
|
|
|
|
d0 = D(0)
|
|
# Move all the above into generation 2.
|
|
gc.collect()
|
|
|
|
c1 = C1055820(1)
|
|
c1.keep_d0_alive = d0
|
|
del d0.loop # now only c1 keeps d0 alive
|
|
|
|
c2 = C1055820(2)
|
|
c2wr = weakref.ref(c2) # no callback!
|
|
|
|
d0 = c1 = c2 = None
|
|
|
|
# What we've set up: d0, c1, and c2 are all trash now. d0 is in
|
|
# generation 2. The only thing keeping it alive is that c1 points to it.
|
|
# c1 and c2 are in generation 0, and are in self-loops. There's a global
|
|
# weakref to c2 (c2wr), but that weakref has no callback. There are no
|
|
# other weakrefs.
|
|
#
|
|
# d0 has a __del__ method that references c2wr
|
|
# ^
|
|
# |
|
|
# | Generation 2 above dots
|
|
#. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
|
|
# | Generation 0 below dots
|
|
# |
|
|
# |
|
|
# ^->c1 ^->c2 has a wr but no callback
|
|
# | | | |
|
|
# <--v <--v
|
|
#
|
|
# So this is the nightmare: when generation 0 gets collected, we see that
|
|
# c2 has a callback-free weakref, and c1 doesn't even have a weakref.
|
|
# Collecting generation 0 doesn't see d0 at all. gc clears c1 and c2.
|
|
# Clearing c1 has the side effect of dropping the refcount on d0 to 0, so
|
|
# d0 goes away (despite that it's in an older generation) and d0's __del__
|
|
# triggers. That in turn materializes a reference to c2 via c2wr(), but
|
|
# c2 gets cleared anyway by gc.
|
|
|
|
# We want to let gc happen "naturally", to preserve the distinction
|
|
# between generations.
|
|
detector = GC_Detector()
|
|
junk = []
|
|
i = 0
|
|
while not detector.gc_happened:
|
|
i += 1
|
|
if i > 10000:
|
|
raise TestFailed("gc didn't happen after 10000 iterations")
|
|
expect(len(ouch), 0, "bug1055820d")
|
|
junk.append([]) # this will eventually trigger gc
|
|
|
|
expect(len(ouch), 1, "bug1055820d") # else __del__ wasn't invoked
|
|
for x in ouch:
|
|
# If __del__ resurrected c2, the instance would be damaged, with an
|
|
# empty __dict__.
|
|
expect(x, None, "bug1055820d")
|
|
|
|
|
|
def test_all():
|
|
gc.collect() # Delete 2nd generation garbage
|
|
run_test("lists", test_list)
|
|
run_test("dicts", test_dict)
|
|
run_test("tuples", test_tuple)
|
|
run_test("classes", test_class)
|
|
run_test("new style classes", test_newstyleclass)
|
|
run_test("instances", test_instance)
|
|
run_test("new instances", test_newinstance)
|
|
run_test("methods", test_method)
|
|
run_test("functions", test_function)
|
|
run_test("frames", test_frame)
|
|
run_test("finalizers", test_finalizer)
|
|
run_test("finalizers (new class)", test_finalizer_newclass)
|
|
run_test("__del__", test_del)
|
|
run_test("__del__ (new class)", test_del_newclass)
|
|
run_test("saveall", test_saveall)
|
|
run_test("trashcan", test_trashcan)
|
|
run_test("boom", test_boom)
|
|
run_test("boom2", test_boom2)
|
|
run_test("boom_new", test_boom_new)
|
|
run_test("boom2_new", test_boom2_new)
|
|
run_test("get_referents", test_get_referents)
|
|
run_test("bug1055820b", test_bug1055820b)
|
|
|
|
gc.enable()
|
|
try:
|
|
run_test("bug1055820c", test_bug1055820c)
|
|
finally:
|
|
gc.disable()
|
|
|
|
gc.enable()
|
|
try:
|
|
run_test("bug1055820d", test_bug1055820d)
|
|
finally:
|
|
gc.disable()
|
|
|
|
def test():
|
|
if verbose:
|
|
print "disabling automatic collection"
|
|
enabled = gc.isenabled()
|
|
gc.disable()
|
|
verify(not gc.isenabled())
|
|
debug = gc.get_debug()
|
|
gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak
|
|
|
|
try:
|
|
test_all()
|
|
finally:
|
|
gc.set_debug(debug)
|
|
# test gc.enable() even if GC is disabled by default
|
|
if verbose:
|
|
print "restoring automatic collection"
|
|
# make sure to always test gc.enable()
|
|
gc.enable()
|
|
verify(gc.isenabled())
|
|
if not enabled:
|
|
gc.disable()
|
|
|
|
|
|
test()
|