From 1b77cdbde71c0c3925def1d71602cda2070c1341 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 13 Apr 2025 13:34:15 -0400 Subject: [PATCH 01/10] Add set_trace_async for async support --- Lib/pdb.py | 99 +++++++++++++++++++++++++++++++++++++++++++- Lib/test/test_pdb.py | 64 ++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 160a7043a30c55..4253b876184d91 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -378,6 +378,9 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + self.async_shim_frame = None + self.async_awaitable = None + self._chained_exceptions = tuple() self._chained_exception_index = 0 @@ -393,6 +396,57 @@ def set_trace(self, frame=None, *, commands=None): super().set_trace(frame) + async def set_trace_async(self, frame=None, *, commands=None): + if self.async_awaitable is not None: + # We are already in a set_trace_async call, do not mess with it + return + + if frame is None: + frame = sys._getframe().f_back + + # We need set_trace to set up the basics, however, this will call + # set_stepinstr() will we need to compensate for, because we don't + # want to trigger on calls + self.set_trace(frame, commands=commands) + # Changing the stopframe will disable trace dispatch on calls + self.stopframe = frame + # We need to stop tracing because we don't have the privilege to avoid + # triggering tracing functions as normal, as we are not already in + # tracing functions + self.stop_trace() + + self.async_shim_frame = sys._getframe() + self.async_awaitable = None + + while True: + self.async_awaitable = None + # Simulate a trace event + # This should bring up pdb and make pdb believe it's debugging the + # caller frame + self.trace_dispatch(frame, "opcode", None) + if self.async_awaitable is not None: + try: + if self.breaks: + with self.set_enterframe(frame): + # set_continue requires enterframe to work + self.set_continue() + self.start_trace() + await self.async_awaitable + except Exception: + self._error_exc() + else: + break + + self.async_shim_frame = None + + # start the trace (the actual command is already set by set_* calls) + if self.returnframe is None and self.stoplineno == -1 and not self.breaks: + # This means we did a continue without any breakpoints, we should not + # start the trace + return + + self.start_trace() + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -775,6 +829,20 @@ def _exec_in_closure(self, source, globals, locals): return True + def _exec_await(self, source, globals, locals): + """ Run source code that contains await by playing with async shim frame""" + # Put the source in an async function + source_async = ( + "async def __pdb_await():\n" + + textwrap.indent(source, " ") + '\n' + + " __pdb_locals.update(locals())" + ) + ns = globals | locals + # We use __pdb_locals to do write back + ns["__pdb_locals"] = locals + exec(source_async, ns) + self.async_awaitable = ns["__pdb_await"]() + def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe.f_locals @@ -820,8 +888,20 @@ def default(self, line): sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook - except: - self._error_exc() + except Exception as e: + # Maybe it's an await expression/statement + if ( + isinstance(e, SyntaxError) + and e.msg == "'await' outside function" + and self.async_shim_frame is not None + ): + try: + self._exec_await(buffer, globals, locals) + return True + except: + self._error_exc() + else: + self._error_exc() def _replace_convenience_variables(self, line): """Replace the convenience variables in 'line' with their values. @@ -2491,6 +2571,21 @@ def set_trace(*, header=None, commands=None): pdb.message(header) pdb.set_trace(sys._getframe().f_back, commands=commands) +async def set_trace_async(*, header=None, commands=None): + """Enter the debugger at the calling stack frame, but in async mode. + + This should be used as await pdb.set_trace_async(). Users can do await + if they enter the debugger with this function. Otherwise it's the same + as set_trace(). + """ + if Pdb._last_pdb_instance is not None: + pdb = Pdb._last_pdb_instance + else: + pdb = Pdb(mode='inline', backend='monitoring') + if header is not None: + pdb.message(header) + await pdb.set_trace_async(sys._getframe().f_back, commands=commands) + # Post-Mortem interface def post_mortem(t=None): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 18fb94af479527..396d0e0492a2f7 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2169,6 +2169,70 @@ def test_pdb_asynctask(): (Pdb) continue """ + def test_pdb_await_support(): + """Testing await support in pdb + + >>> import asyncio + + >>> async def test(): + ... print("hello") + ... await asyncio.sleep(0) + ... print("world") + ... return 42 + + >>> async def main(): + ... import pdb; + ... task = asyncio.create_task(test()) + ... await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + ... pass + + >>> def test_function(): + ... asyncio.run(main(), loop_factory=asyncio.EventLoop) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'x = await task', + ... 'p x', + ... 'x = await test()', + ... 'p x', + ... 'new_task = asyncio.create_task(test())', + ... 'await new_task', + ... 'await non_exist()', + ... 's', + ... 'continue', + ... ]): + ... test_function() + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) x = await task + hello + world + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p x + 42 + (Pdb) x = await test() + hello + world + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p x + 42 + (Pdb) new_task = asyncio.create_task(test()) + (Pdb) await new_task + hello + world + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) await non_exist() + *** NameError: name 'non_exist' is not defined + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) s + > (5)main() + -> pass + (Pdb) continue + """ + def test_pdb_next_command_for_coroutine(): """Testing skip unwinding stack on yield for coroutines for "next" command From 677c95205619359eb04aea93ffbdb864a4bce054 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 15 Apr 2025 21:35:20 -0400 Subject: [PATCH 02/10] Update docs --- Doc/library/pdb.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 8601f390aeb9c9..3c8c07074993f2 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -188,6 +188,21 @@ slightly different way: .. versionadded:: 3.14 The *commands* argument. + +.. awaitablefunction:: set_trace_async(*, header=None, commands=None) + + async version of :func:`set_trace`. This function should be used inside an + async function with :keyword:`await`. + + .. code-block:: python + + async def f(): + await pdb.set_trace_async() + + :keyword:`await` statements are supported if the debugger is invoked by this function. + + .. versionadded:: 3.14 + .. function:: post_mortem(t=None) Enter post-mortem debugging of the given exception or From 15290d51a44e966eb479117eebcc4c7dac05d39e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:41:37 +0000 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-04-16-01-41-34.gh-issue-121468.rxgE1z.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-16-01-41-34.gh-issue-121468.rxgE1z.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-16-01-41-34.gh-issue-121468.rxgE1z.rst b/Misc/NEWS.d/next/Library/2025-04-16-01-41-34.gh-issue-121468.rxgE1z.rst new file mode 100644 index 00000000000000..a46db6b73b7945 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-16-01-41-34.gh-issue-121468.rxgE1z.rst @@ -0,0 +1 @@ +Add :func:`pdb.set_trace_async` function to support :keyword:`await` statements in :mod:`pdb`. From a3401e92754c148abd956f619d9829bc98a5c748 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 15 Apr 2025 21:42:33 -0400 Subject: [PATCH 04/10] reorder the if checks --- Lib/pdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 4253b876184d91..137756ed30e447 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -891,9 +891,9 @@ def default(self, line): except Exception as e: # Maybe it's an await expression/statement if ( - isinstance(e, SyntaxError) + self.async_shim_frame is not None + and isinstance(e, SyntaxError) and e.msg == "'await' outside function" - and self.async_shim_frame is not None ): try: self._exec_await(buffer, globals, locals) From 50c3c9c5117309209a40c9d4ef313b257c0875f4 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 15 Apr 2025 21:51:20 -0400 Subject: [PATCH 05/10] Add breakpoint tests --- Lib/test/test_pdb.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 05c72b90cc46ad..154676f49c1e82 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2233,6 +2233,59 @@ def test_pdb_await_support(): (Pdb) continue """ + def test_pdb_await_with_breakpoint(): + """Testing await support with breakpoints set in tasks + + >>> reset_Breakpoint() + + >>> import asyncio + + >>> async def test(): + ... x = 2 + ... await asyncio.sleep(0) + ... return 42 + + >>> async def main(): + ... import pdb; + ... task = asyncio.create_task(test()) + ... await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + + >>> def test_function(): + ... asyncio.run(main(), loop_factory=asyncio.EventLoop) + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'b test', + ... 'k = await task', + ... 'n', + ... 'p x', + ... 'continue', + ... 'p k', + ... 'clear 1', + ... 'continue', + ... ]): + ... test_function() + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) b test + Breakpoint 1 at :2 + (Pdb) k = await task + > (2)test() + -> x = 2 + (Pdb) n + > (3)test() + -> await asyncio.sleep(0) + (Pdb) p x + 2 + (Pdb) continue + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p k + 42 + (Pdb) clear 1 + Deleted breakpoint 1 at :2 + (Pdb) continue + """ + def test_pdb_next_command_for_coroutine(): """Testing skip unwinding stack on yield for coroutines for "next" command From 090715bdaa7005e757262f8efbf14c29480e0d84 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 16 Apr 2025 19:41:25 -0400 Subject: [PATCH 06/10] Update test --- Lib/test/test_pdb.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index cc4c912adfb50a..a59f34856b2f33 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1,6 +1,7 @@ # A test suite for pdb; not very comprehensive at the moment. import doctest +import gc import os import pdb import sys @@ -2209,9 +2210,7 @@ def test_pdb_await_support(): def test_pdb_await_with_breakpoint(): """Testing await support with breakpoints set in tasks - >>> reset_Breakpoint() - - >>> import asyncio + >>> import asyncio >>> async def test(): ... x = 2 @@ -2233,29 +2232,26 @@ def test_pdb_await_with_breakpoint(): ... 'p x', ... 'continue', ... 'p k', - ... 'clear 1', ... 'continue', ... ]): ... test_function() - > (4)main() + > (4)main() -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() (Pdb) b test - Breakpoint 1 at :2 + Breakpoint 1 at :2 (Pdb) k = await task - > (2)test() + > (2)test() -> x = 2 (Pdb) n - > (3)test() + > (3)test() -> await asyncio.sleep(0) (Pdb) p x 2 (Pdb) continue - > (4)main() + > (4)main() -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() (Pdb) p k 42 - (Pdb) clear 1 - Deleted breakpoint 1 at :2 (Pdb) continue """ @@ -4815,6 +4811,10 @@ def tearDown(test): pdb.Pdb._last_pdb_instance.stop_trace() pdb.Pdb._last_pdb_instance = None + # If garbage objects are collected right after we start tracing, we + # could stop at __del__ of the object which would fail the test. + gc.collect() + tests.addTest( doctest.DocTestSuite( test_pdb, From ebdc161d207e142494779318051772ffad62f49f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 16 Apr 2025 19:41:33 -0400 Subject: [PATCH 07/10] Add whatsnew entry --- Doc/whatsnew/3.14.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index c50d1669fef84c..d74dbdf1e83f36 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1019,6 +1019,11 @@ pdb backend by default, which is configurable. (Contributed by Tian Gao in :gh:`124533`.) +* :func:`pdb.set_trace_async` is added to support debugging asyncio + coroutines. :keyword:`await` statements are supported with this + function. + (Contributed by Tian Gao in :gh:`132576`.) + pickle ------ From 109d8b1632c1ac25961fbd8d5384fd2acf10a260 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 22 Apr 2025 11:55:03 -0400 Subject: [PATCH 08/10] Restructure default function --- Lib/pdb.py | 57 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 137756ed30e447..3be61c8a0f1d78 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -843,12 +843,11 @@ def _exec_await(self, source, globals, locals): exec(source_async, ns) self.async_awaitable = ns["__pdb_await"]() - def default(self, line): - if line[:1] == '!': line = line[1:].strip() - locals = self.curframe.f_locals - globals = self.curframe.f_globals + def _read_code(self, line): + buffer = line + is_await_code = False + code = None try: - buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode with self._enable_multiline_completion(): @@ -861,7 +860,7 @@ def default(self, line): except (EOFError, KeyboardInterrupt): self.lastcmd = "" print('\n') - return + return None, None, False else: self.stdout.write(continue_prompt) self.stdout.flush() @@ -870,11 +869,31 @@ def default(self, line): self.lastcmd = "" self.stdout.write('\n') self.stdout.flush() - return + return None, None, False else: line = line.rstrip('\r\n') buffer += '\n' + line self.lastcmd = buffer + except SyntaxError as e: + # Maybe it's an await expression/statement + if ( + self.async_shim_frame is not None + and e.msg == "'await' outside function" + ): + is_await_code = True + else: + raise + + return code, buffer, is_await_code + + def default(self, line): + if line[:1] == '!': line = line[1:].strip() + locals = self.curframe.f_locals + globals = self.curframe.f_globals + try: + code, buffer, is_await_code = self._read_code(line) + if buffer is None: + return save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook @@ -882,26 +901,18 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - if not self._exec_in_closure(buffer, globals, locals): - exec(code, globals, locals) + if is_await_code: + self._exec_await(buffer, globals, locals) + return True + else: + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook - except Exception as e: - # Maybe it's an await expression/statement - if ( - self.async_shim_frame is not None - and isinstance(e, SyntaxError) - and e.msg == "'await' outside function" - ): - try: - self._exec_await(buffer, globals, locals) - return True - except: - self._error_exc() - else: - self._error_exc() + except: + self._error_exc() def _replace_convenience_variables(self, line): """Replace the convenience variables in 'line' with their values. From e95353206ed297269f700cfa6a7972704a9d85fa Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 22 Apr 2025 17:09:54 -0400 Subject: [PATCH 09/10] Add context var tests --- Lib/test/test_pdb.py | 60 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index a59f34856b2f33..930cc36ac4603c 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2155,7 +2155,7 @@ def test_pdb_await_support(): ... return 42 >>> async def main(): - ... import pdb; + ... import pdb ... task = asyncio.create_task(test()) ... await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() ... pass @@ -2218,7 +2218,7 @@ def test_pdb_await_with_breakpoint(): ... return 42 >>> async def main(): - ... import pdb; + ... import pdb ... task = asyncio.create_task(test()) ... await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() @@ -2255,6 +2255,62 @@ def test_pdb_await_with_breakpoint(): (Pdb) continue """ + def test_pdb_await_contextvar(): + """Testing await support context vars + + >>> import asyncio + >>> import contextvars + + >>> var = contextvars.ContextVar('var') + + >>> async def get_var(): + ... return var.get() + + >>> async def set_var(val): + ... var.set(val) + ... return var.get() + + >>> async def main(): + ... var.set(42) + ... import pdb + ... await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + + >>> def test_function(): + ... asyncio.run(main(), loop_factory=asyncio.EventLoop) + + >>> with PdbTestInput([ + ... 'p var.get()', + ... 'print(await get_var())', + ... 'print(await asyncio.create_task(set_var(100)))', + ... 'p var.get()', + ... 'print(await set_var(99))', + ... 'p var.get()', + ... 'continue', + ... ]): + ... test_function() + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p var.get() + 42 + (Pdb) print(await get_var()) + 42 + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) print(await asyncio.create_task(set_var(100))) + 100 + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p var.get() + 42 + (Pdb) print(await set_var(99)) + 99 + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() + (Pdb) p var.get() + 99 + (Pdb) continue + """ + def test_pdb_next_command_for_coroutine(): """Testing skip unwinding stack on yield for coroutines for "next" command From ff09ab25edf086912c9fbe603e536cacbd3ba448 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 25 Apr 2025 13:51:28 -0400 Subject: [PATCH 10/10] Add an extra check for get_var() --- Lib/test/test_pdb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 6cce6b7d05b4a6..cc2c0bea940d1b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2285,6 +2285,7 @@ def test_pdb_await_contextvar(): ... 'p var.get()', ... 'print(await set_var(99))', ... 'p var.get()', + ... 'print(await get_var())', ... 'continue', ... ]): ... test_function() @@ -2308,6 +2309,10 @@ def test_pdb_await_contextvar(): -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() (Pdb) p var.get() 99 + (Pdb) print(await get_var()) + 99 + > (4)main() + -> await pdb.Pdb(nosigint=True, readrc=False).set_trace_async() (Pdb) continue """