File size: 7,337 Bytes
d1ceb73 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 |
"""Test script to find circular references.
Circular references are not leaks per se, because they will eventually
be GC'd. However, on CPython, they prevent the reference-counting fast
path from being used and instead rely on the slower full GC. This
increases memory footprint and CPU overhead, so we try to eliminate
circular references created by normal operation.
"""
import asyncio
import contextlib
import gc
import io
import sys
import traceback
import types
import typing
import unittest
import tornado
from tornado import web, gen, httpclient
from tornado.test.util import skipNotCPython
def find_circular_references(garbage):
"""Find circular references in a list of objects.
The garbage list contains objects that participate in a cycle,
but also the larger set of objects kept alive by that cycle.
This function finds subsets of those objects that make up
the cycle(s).
"""
def inner(level):
for item in level:
item_id = id(item)
if item_id not in garbage_ids:
continue
if item_id in visited_ids:
continue
if item_id in stack_ids:
candidate = stack[stack.index(item) :]
candidate.append(item)
found.append(candidate)
continue
stack.append(item)
stack_ids.add(item_id)
inner(gc.get_referents(item))
stack.pop()
stack_ids.remove(item_id)
visited_ids.add(item_id)
found: typing.List[object] = []
stack = []
stack_ids = set()
garbage_ids = set(map(id, garbage))
visited_ids = set()
inner(garbage)
return found
@contextlib.contextmanager
def assert_no_cycle_garbage():
"""Raise AssertionError if the wrapped code creates garbage with cycles."""
gc.disable()
gc.collect()
gc.set_debug(gc.DEBUG_STATS | gc.DEBUG_SAVEALL)
yield
try:
# We have DEBUG_STATS on which causes gc.collect to write to stderr.
# Capture the output instead of spamming the logs on passing runs.
f = io.StringIO()
old_stderr = sys.stderr
sys.stderr = f
try:
gc.collect()
finally:
sys.stderr = old_stderr
garbage = gc.garbage[:]
# Must clear gc.garbage (the same object, not just replacing it with a
# new list) to avoid warnings at shutdown.
gc.garbage[:] = []
if len(garbage) == 0:
return
for circular in find_circular_references(garbage):
f.write("\n==========\n Circular \n==========")
for item in circular:
f.write(f"\n {repr(item)}")
for item in circular:
if isinstance(item, types.FrameType):
f.write(f"\nLocals: {item.f_locals}")
f.write(f"\nTraceback: {repr(item)}")
traceback.print_stack(item)
del garbage
raise AssertionError(f.getvalue())
finally:
gc.set_debug(0)
gc.enable()
# GC behavior is cpython-specific
@skipNotCPython
class CircleRefsTest(unittest.TestCase):
def test_known_leak(self):
# Construct a known leak scenario to make sure the test harness works.
class C(object):
def __init__(self, name):
self.name = name
self.a: typing.Optional[C] = None
self.b: typing.Optional[C] = None
self.c: typing.Optional[C] = None
def __repr__(self):
return f"name={self.name}"
with self.assertRaises(AssertionError) as cm:
with assert_no_cycle_garbage():
# a and b form a reference cycle. c is not part of the cycle,
# but it cannot be GC'd while a and b are alive.
a = C("a")
b = C("b")
c = C("c")
a.b = b
a.c = c
b.a = a
b.c = c
del a, b
self.assertIn("Circular", str(cm.exception))
# Leading spaces ensure we only catch these at the beginning of a line, meaning they are a
# cycle participant and not simply the contents of a locals dict or similar container. (This
# depends on the formatting above which isn't ideal but this test evolved from a
# command-line script) Note that the behavior here changed in python 3.11; in newer pythons
# locals are handled a bit differently and the test passes without the spaces.
self.assertIn(" name=a", str(cm.exception))
self.assertIn(" name=b", str(cm.exception))
self.assertNotIn(" name=c", str(cm.exception))
async def run_handler(self, handler_class):
app = web.Application(
[
(r"/", handler_class),
]
)
socket, port = tornado.testing.bind_unused_port()
server = tornado.httpserver.HTTPServer(app)
server.add_socket(socket)
client = httpclient.AsyncHTTPClient()
with assert_no_cycle_garbage():
# Only the fetch (and the corresponding server-side handler)
# are being tested for cycles. In particular, the Application
# object has internal cycles (as of this writing) which we don't
# care to fix since in real world usage the Application object
# is effectively a global singleton.
await client.fetch(f"http://127.0.0.1:{port}/")
client.close()
server.stop()
socket.close()
def test_sync_handler(self):
class Handler(web.RequestHandler):
def get(self):
self.write("ok\n")
asyncio.run(self.run_handler(Handler))
def test_finish_exception_handler(self):
class Handler(web.RequestHandler):
def get(self):
raise web.Finish("ok\n")
asyncio.run(self.run_handler(Handler))
def test_coro_handler(self):
class Handler(web.RequestHandler):
@gen.coroutine
def get(self):
yield asyncio.sleep(0.01)
self.write("ok\n")
asyncio.run(self.run_handler(Handler))
def test_async_handler(self):
class Handler(web.RequestHandler):
async def get(self):
await asyncio.sleep(0.01)
self.write("ok\n")
asyncio.run(self.run_handler(Handler))
def test_run_on_executor(self):
# From https://github.com/tornadoweb/tornado/issues/2620
#
# When this test was introduced it found cycles in IOLoop.add_future
# and tornado.concurrent.chain_future.
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(1) as thread_pool:
class Factory(object):
executor = thread_pool
@tornado.concurrent.run_on_executor
def run(self):
return None
factory = Factory()
async def main():
# The cycle is not reported on the first call. It's not clear why.
for i in range(2):
await factory.run()
with assert_no_cycle_garbage():
asyncio.run(main())
|