Skip to content

bpo-31861: Provide aiter and anext builtins #23847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 41 commits into from
Mar 23, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
44b2a60
bpo-31861: Add operator.aiter and operator.anext
jab Aug 24, 2018
142523b
address comments from first review
jab Sep 7, 2018
3ebce05
Address new review comments + rebase
jab Nov 30, 2020
2f8df99
improve tests
jab Dec 4, 2020
ad12116
Sketch out async iterator in C
lordmauve Dec 4, 2020
ce35092
Implement aiter() and anext() using sync methods only
lordmauve Dec 7, 2020
f9dc183
Add basic aiter built-in [WIP]
justin39 Dec 7, 2020
086bb79
Start aiter implementation
justin39 Dec 9, 2020
29ef712
Get test_asyncgen tests passing
justin39 Dec 10, 2020
331e80e
Fix awaitable iternext
justin39 Dec 10, 2020
95879d8
Add anext builtin
justin39 Dec 11, 2020
5b64589
Use stop iter functions for anext
justin39 Dec 14, 2020
6e145db
Use stop iter functions in aiter
justin39 Dec 14, 2020
2fd4be6
Note about implementing __reduce__ for aiter
justin39 Dec 16, 2020
430dd59
Refactor aiter and anext type names
justin39 Dec 16, 2020
4266e65
Add documentation for aiter()
justin39 Dec 16, 2020
ecbedc7
Update documentation for aiter and anext
justin39 Dec 18, 2020
5fa3812
Clean up docs and formatting
justin39 Dec 18, 2020
0f9c814
Remove async iterator code from operator.py
justin39 Dec 18, 2020
8160c82
Cleanup formatting
justin39 Dec 18, 2020
fa8b12a
Fix test_builtins_have_signatures + misc. cleanups
jab Dec 18, 2020
a2242d9
Use PyErr_SetNone now that we can
jab Dec 18, 2020
3ce5675
whitespace
jab Dec 18, 2020
0038cde
cosmetic fixes
jab Dec 19, 2020
06019ea
Add null check and use StopAsyncIteration when aiter is exhausted
justin39 Dec 19, 2020
0cc00f2
Fix AC definition and resulting signature
justin39 Dec 21, 2020
8b8a689
Fix comparison to NULL instead of Py_None
justin39 Dec 21, 2020
894600d
Revert None deafult for aiter/anext and add whatsnew entry
justin39 Dec 21, 2020
e687d88
Merge branch 'master' of https://github.com/python/cpython into justi…
justin39 Feb 26, 2021
259af97
Delint tests + docs (fixes CI), alphabetize names.
jab Mar 20, 2021
97d06e5
Merge branch 'master' into justin39/aiter-c
jab Mar 20, 2021
dd7c02d
Fix code style and comments.
jab Mar 20, 2021
2fbdd5b
Remove 2-arg variant of aiter.
jab Mar 22, 2021
009118c
Fix use of clinic.
jab Mar 22, 2021
71fede3
Address some feedback from Guido's review.
jab Mar 22, 2021
4a51ace
No longer need to exclude aiter from test_builtins_have_signatures.
jab Mar 22, 2021
108e4a3
Improve aiter() docs.
jab Mar 22, 2021
042f596
Fix typo.
jab Mar 22, 2021
6ee8824
Add test_aiter_idempotent().
jab Mar 23, 2021
6f50ef8
Remove public API added for anext.
jab Mar 23, 2021
ef40fb7
Slightly improve wording
gvanrossum Mar 23, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 50 additions & 25 deletions Doc/library/functions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,31 @@ are always available. They are listed here in alphabetical order.
+=========================+=======================+=======================+=========================+
| | **A** | | **E** | | **L** | | **R** |
| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ |
| | :func:`all` | | :func:`eval` | | |func-list|_ | | :func:`repr` |
| | :func:`any` | | :func:`exec` | | :func:`locals` | | :func:`reversed` |
| | :func:`ascii` | | | | | | :func:`round` |
| | | | **F** | | **M** | | |
| | **B** | | :func:`filter` | | :func:`map` | | **S** |
| | :func:`bin` | | :func:`float` | | :func:`max` | | |func-set|_ |
| | :func:`bool` | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` |
| | :func:`breakpoint` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` |
| | |func-bytearray|_ | | | | | | :func:`sorted` |
| | |func-bytes|_ | | **G** | | **N** | | :func:`staticmethod` |
| | | | :func:`getattr` | | :func:`next` | | |func-str|_ |
| | **C** | | :func:`globals` | | | | :func:`sum` |
| | :func:`callable` | | | | **O** | | :func:`super` |
| | :func:`chr` | | **H** | | :func:`object` | | |
| | :func:`classmethod` | | :func:`hasattr` | | :func:`oct` | | **T** |
| | :func:`compile` | | :func:`hash` | | :func:`open` | | |func-tuple|_ |
| | :func:`complex` | | :func:`help` | | :func:`ord` | | :func:`type` |
| | | | :func:`hex` | | | | |
| | **D** | | | | **P** | | **V** |
| | :func:`delattr` | | **I** | | :func:`pow` | | :func:`vars` |
| | |func-dict|_ | | :func:`id` | | :func:`print` | | |
| | :func:`dir` | | :func:`input` | | :func:`property` | | **Z** |
| | :func:`divmod` | | :func:`int` | | | | :func:`zip` |
| | | | :func:`isinstance` | | | | |
| | | | :func:`issubclass` | | | | **_** |
| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` |
| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` |
| | :func:`any` | | | | | | :func:`round` |
| | :func:`anext` | | **F** | | **M** | | |
| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** |
| | | | :func:`float` | | :func:`max` | | |func-set|_ |
| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` |
| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` |
| | :func:`bool` | | | | | | :func:`sorted` |
| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` |
| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ |
| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` |
| | | | | | **O** | | :func:`super` |
| | **C** | | **H** | | :func:`object` | | |
| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** |
| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ |
| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` |
| | :func:`compile` | | :func:`hex` | | | | |
| | :func:`complex` | | | | **P** | | **V** |
| | | | **I** | | :func:`pow` | | :func:`vars` |
| | **D** | | :func:`id` | | :func:`print` | | |
| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** |
| | |func-dict|_ | | :func:`int` | | | | :func:`zip` |
| | :func:`dir` | | :func:`isinstance` | | | | |
| | :func:`divmod` | | :func:`issubclass` | | | | **_** |
| | | | :func:`iter` | | | | :func:`__import__` |
+-------------------------+-----------------------+-----------------------+-------------------------+

Expand All @@ -61,6 +61,17 @@ are always available. They are listed here in alphabetical order.
If the argument is a complex number, its magnitude is returned.


.. function:: aiter(async_iterable)

Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`.
Equivalent to calling ``x.__aiter__()``.

``aiter(x)`` itself has an ``__aiter__()`` method that returns ``x``,
so ``aiter(aiter(x))`` is the same as ``aiter(x)``.

Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant.


.. function:: all(iterable)

Return ``True`` if all elements of the *iterable* are true (or if the iterable
Expand All @@ -73,6 +84,20 @@ are always available. They are listed here in alphabetical order.
return True


.. awaitablefunction:: anext(async_iterator[, default])

When awaited, return the next item from the given :term:`asynchronous
iterator`, or *default* if given and the iterator is exhausted.

This is the async variant of the :func:`next` builtin, and behaves
similarly.

This calls the :meth:`~object.__anext__` method of *async_iterator*,
returning an :term:`awaitable`. Awaiting this returns the next value of the
iterator. If *default* is given, it is returned if the iterator is exhausted,
otherwise :exc:`StopAsyncIteration` is raised.


.. function:: any(iterable)

Return ``True`` if any element of the *iterable* is true. If the iterable
Expand Down
5 changes: 5 additions & 0 deletions Doc/whatsnew/3.10.rst
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,11 @@ Other Language Changes
``__globals__["__builtins__"]`` if it exists, else from the current builtins.
(Contributed by Mark Shannon in :issue:`42990`.)

* Two new builtin functions -- :func:`aiter` and :func:`anext` have been added
to provide asynchronous counterparts to :func:`iter` and :func:`next`,
respectively.
(Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.)


New Modules
===========
Expand Down
10 changes: 10 additions & 0 deletions Include/abstract.h
Original file line number Diff line number Diff line change
Expand Up @@ -324,11 +324,21 @@ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj,
returns itself. */
PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *);

/* Takes an AsyncIterable object and returns an AsyncIterator for it.
This is typically a new iterator but if the argument is an AsyncIterator,
this returns itself. */
PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it necessary to add these to the public C API? Just because PyObject_GetIter() is public I'm not sure that the Aiter variant needs to be. @vstinner tends to push back on adding new things to the C API. @1st1 what do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't mind adding this function -- while somewhat trivial, it's something that projects like Cython (and potentially our own modules like _asynciomodule.c) have to reimplement.


/* Returns non-zero if the object 'obj' provides iterator protocols, and 0 otherwise.

This function always succeeds. */
PyAPI_FUNC(int) PyIter_Check(PyObject *);

/* Returns non-zero if the object 'obj' provides AsyncIterator protocols, and 0 otherwise.

This function always succeeds. */
PyAPI_FUNC(int) PyAiter_Check(PyObject *);

/* Takes an iterator object and calls its tp_iternext slot,
returning the next value.

Expand Down
82 changes: 82 additions & 0 deletions Lib/test/test_asyncgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,88 @@ def tearDown(self):
self.loop = None
asyncio.set_event_loop_policy(None)

def test_async_gen_anext(self):
async def gen():
yield 1
yield 2
g = gen()
async def consume():
results = []
results.append(await anext(g))
results.append(await anext(g))
results.append(await anext(g, 'buckle my shoe'))
return results
res = self.loop.run_until_complete(consume())
self.assertEqual(res, [1, 2, 'buckle my shoe'])
with self.assertRaises(StopAsyncIteration):
self.loop.run_until_complete(consume())

def test_async_gen_aiter(self):
async def gen():
yield 1
yield 2
g = gen()
async def consume():
return [i async for i in aiter(g)]
res = self.loop.run_until_complete(consume())
self.assertEqual(res, [1, 2])

def test_async_gen_aiter_class(self):
results = []
class Gen:
async def __aiter__(self):
yield 1
yield 2
g = Gen()
async def consume():
ait = aiter(g)
while True:
try:
results.append(await anext(ait))
except StopAsyncIteration:
break
self.loop.run_until_complete(consume())
self.assertEqual(results, [1, 2])

def test_aiter_idempotent(self):
async def gen():
yield 1
applied_once = aiter(gen())
applied_twice = aiter(applied_once)
self.assertIs(applied_once, applied_twice)

def test_anext_bad_args(self):
async def gen():
yield 1
async def call_with_too_few_args():
await anext()
async def call_with_too_many_args():
await anext(gen(), 1, 3)
async def call_with_wrong_type_args():
await anext(1, gen())
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_too_few_args())
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_too_many_args())
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_wrong_type_args())

def test_aiter_bad_args(self):
async def gen():
yield 1
async def call_with_too_few_args():
await aiter()
async def call_with_too_many_args():
await aiter(gen(), 1)
async def call_with_wrong_type_arg():
await aiter(1)
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_too_few_args())
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_too_many_args())
with self.assertRaises(TypeError):
self.loop.run_until_complete(call_with_wrong_type_arg())

async def to_list(self, gen):
res = []
async for i in gen:
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -3860,6 +3860,9 @@ def test_builtins_have_signatures(self):
needs_groups = {"range", "slice", "dir", "getattr",
"next", "iter", "vars"}
no_signature |= needs_groups
# These have unrepresentable parameter default values of NULL
needs_null = {"anext"}
no_signature |= needs_null
# These need PEP 457 groups or a signature change to accept None
needs_semantic_update = {"round"}
no_signature |= needs_semantic_update
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Add builtins.aiter and builtins.anext.
Patch by Joshua Bronson (@jab), Daniel Pope (@lordmauve), and Justin Wang (@justin39).
31 changes: 31 additions & 0 deletions Objects/abstract.c
Original file line number Diff line number Diff line change
Expand Up @@ -2738,6 +2738,26 @@ PyObject_GetIter(PyObject *o)
}
}

PyObject *
PyObject_GetAiter(PyObject *o) {
PyTypeObject *t = Py_TYPE(o);
unaryfunc f;

if (t->tp_as_async == NULL || t->tp_as_async->am_aiter == NULL) {
return type_error("'%.200s' object is not an AsyncIterable", o);
}
f = t->tp_as_async->am_aiter;
PyObject *it = (*f)(o);
if (it != NULL && !PyAiter_Check(it)) {
PyErr_Format(PyExc_TypeError,
"aiter() returned non-AsyncIterator of type '%.100s'",
Py_TYPE(it)->tp_name);
Py_DECREF(it);
it = NULL;
}
return it;
}

int
PyIter_Check(PyObject *obj)
{
Expand All @@ -2746,6 +2766,17 @@ PyIter_Check(PyObject *obj)
tp->tp_iternext != &_PyObject_NextNotImplemented);
}

int
PyAiter_Check(PyObject *obj)
{
PyTypeObject *tp = Py_TYPE(obj);
return (tp->tp_as_async != NULL &&
tp->tp_as_async->am_aiter != NULL &&
tp->tp_as_async->am_aiter != &_PyObject_NextNotImplemented &&
tp->tp_as_async->am_anext != NULL &&
tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented);
}

/* Return next item.
* If an error occurs, return NULL. PyErr_Occurred() will be true.
* If the iteration terminates normally, return NULL and clear the
Expand Down
92 changes: 90 additions & 2 deletions Objects/iterobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ PyTypeObject PySeqIter_Type = {
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
0, /* tp_doc */
(traverseproc)iter_traverse, /* tp_traverse */
0, /* tp_clear */
Expand Down Expand Up @@ -276,7 +276,7 @@ PyTypeObject PyCallIter_Type = {
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
0, /* tp_doc */
(traverseproc)calliter_traverse, /* tp_traverse */
0, /* tp_clear */
Expand All @@ -288,3 +288,91 @@ PyTypeObject PyCallIter_Type = {
};


/* -------------------------------------- */

typedef struct {
PyObject_HEAD
PyObject *wrapped;
PyObject *default_value;
} anextawaitableobject;

static void
anextawaitable_dealloc(anextawaitableobject *obj)
{
_PyObject_GC_UNTRACK(obj);
Py_XDECREF(obj->wrapped);
Py_XDECREF(obj->default_value);
PyObject_GC_Del(obj);
}

static int
anextawaitable_traverse(anextawaitableobject *obj, visitproc visit, void *arg)
{
Py_VISIT(obj->wrapped);
Py_VISIT(obj->default_value);
return 0;
}

static PyObject *
anextawaitable_iternext(anextawaitableobject *obj)
{
PyObject *result = PyIter_Next(obj->wrapped);
if (result != NULL) {
return result;
}
if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) {
_PyGen_SetStopIterationValue(obj->default_value);
}
return NULL;
}

static PyAsyncMethods anextawaitable_as_async = {
PyObject_SelfIter, /* am_await */
0, /* am_aiter */
0, /* am_anext */
0, /* am_send */
};

PyTypeObject PyAnextAwaitable_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"anext_awaitable", /* tp_name */
sizeof(anextawaitableobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)anextawaitable_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
&anextawaitable_as_async, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
0, /* tp_doc */
(traverseproc)anextawaitable_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(unaryfunc)anextawaitable_iternext, /* tp_iternext */
0, /* tp_methods */
};

PyObject *
PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value)
{
anextawaitableobject *anext = PyObject_GC_New(anextawaitableobject, &PyAnextAwaitable_Type);
Py_INCREF(awaitable);
anext->wrapped = awaitable;
Py_INCREF(default_value);
anext->default_value = default_value;
_PyObject_GC_TRACK(anext);
return (PyObject *)anext;
}
Loading