Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Doc/library/asyncio-task.rst
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,10 @@ unless it is :exc:`asyncio.CancelledError`,
is also included in the exception group.
The same special case is made for
:exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph.
There is an additional special case made only for the body of the
``async with``: if it raises :exc:`GeneratorExit` and none of the
other tasks raise exceptions that would be reported, then the
:exc:`GeneratorExit` is reraised.

Task groups are careful not to mix up the internal cancellation used to
"wake up" their :meth:`~object.__aexit__` with cancellation requests
Expand All @@ -456,6 +460,10 @@ reported by :meth:`asyncio.Task.cancelling`.
Improved handling of simultaneous internal and external cancellations
and correct preservation of cancellation counts.

.. versionchanged:: 3.15

Addition of the special case for :exc:`GeneratorExit`.

Sleeping
========

Expand Down
21 changes: 17 additions & 4 deletions Lib/asyncio/taskgroups.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,23 @@ async def _aexit(self, et, exc):
self._parent_task.uncancel()
self._parent_task.cancel()
try:
raise BaseExceptionGroup(
'unhandled errors in a TaskGroup',
self._errors,
) from None
# If the *only* error is a GeneratorExit from the body
# of the group, then instead of raising an
# ExceptionGroup we raise GeneratorExit. This ensures
# that async generators that use TaskGroup properly
# swallow the exception on `aclose()` while ensuring
# that no exceptions from subtasks are swallowed.
if (
et is not None
and issubclass(et, GeneratorExit)
and len(self._errors) == 1
):
raise exc
else:
raise BaseExceptionGroup(
'unhandled errors in a TaskGroup',
self._errors,
) from None
finally:
exc = None

Expand Down
49 changes: 38 additions & 11 deletions Lib/subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -917,6 +917,12 @@ def _can_use_kqueue():
_CAN_USE_PIDFD_OPEN = not _mswindows and _can_use_pidfd_open()
_CAN_USE_KQUEUE = not _mswindows and _can_use_kqueue()

# Maximum timeout passed to poll() / kqueue.control() by Popen._wait():
# very large values (e.g. timeout=float('inf')) overflow the C timestamp
# conversion (gh-154836). Longer waits are performed in bounded slices
# until the deadline, like asyncio's MAXIMUM_SELECT_TIMEOUT.
_MAXIMUM_WAIT_TIMEOUT = 24 * 3600


# These are primarily fail-safe knobs for negatives. A True value does not
# guarantee the given libc/syscall API will be used.
Expand Down Expand Up @@ -2228,10 +2234,19 @@ def _wait_pidfd(self, timeout):
try:
poller = select.poll()
poller.register(pidfd, select.POLLIN)
events = poller.poll(timeout * 1000)
if not events:
raise TimeoutExpired(self.args, timeout)
return True
endtime = _time() + timeout
while True:
# Clamp very large timeouts (e.g. float('inf')):
# they overflow the C timestamp conversion in
# poll(). Wait in bounded slices until the real
# deadline (gh-154836).
delay = min(_deadline_remaining(endtime),
_MAXIMUM_WAIT_TIMEOUT)
events = poller.poll(max(delay, 0) * 1000)
if events:
return True
if _deadline_remaining(endtime) <= 0:
raise TimeoutExpired(self.args, timeout)
finally:
os.close(pidfd)

Expand All @@ -2252,14 +2267,26 @@ def _wait_kqueue(self, timeout):
flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
fflags=select.KQ_NOTE_EXIT,
)
try:
events = kq.control([kev], 1, timeout) # wait
except OSError:
return False
else:
if not events:
changelist = [kev]
endtime = _time() + timeout
while True:
# Clamp very large timeouts (e.g. float('inf')):
# they overflow the C timestamp conversion in
# kqueue.control(). Wait in bounded slices until
# the real deadline (gh-154836).
delay = min(_deadline_remaining(endtime),
_MAXIMUM_WAIT_TIMEOUT)
try:
events = kq.control(changelist, 1, max(delay, 0))
except OSError:
return False
if events:
return True
if _deadline_remaining(endtime) <= 0:
raise TimeoutExpired(self.args, timeout)
return True
# The kevent was registered by the first control()
# call; don't re-add it on later slices.
changelist = None
finally:
kq.close()

Expand Down
66 changes: 66 additions & 0 deletions Lib/test/test_asyncio/test_taskgroups.py
Original file line number Diff line number Diff line change
Expand Up @@ -1227,6 +1227,72 @@ async def fn_3():
self.assertEqual(await race(fn_1, fn_2, fn_3), 1)
self.assertListEqual(record, ["1 started", "2 started", "3 started", "1 finished"])

async def test_taskgroup_generator_exit_01(self):
# GeneratorExit in a TaskGroup should be fine
async def gen():
yield 1

async def fn():
async with asyncio.TaskGroup() as tg:
async for n in gen():
yield n

g = fn()
await g.asend(None)
await g.aclose()

async def test_taskgroup_generator_exit_02(self):
# A lone GeneratorExit in a task should still give an ExceptionGroup
async def t():
raise GeneratorExit

async def fn():
async with asyncio.TaskGroup() as tg:
tg.create_task(t())

with self.assertRaises(BaseExceptionGroup) as cm:
await fn()
self.assertEqual(get_error_types(cm.exception), {GeneratorExit})

async def test_taskgroup_generator_exit_03(self):
# A GeneratorExit in one task and an error in another should
# still give an ExceptionGroup
async def t1():
raise GeneratorExit

async def t2():
raise AssertionError('t2 failed')

async def fn():
async with asyncio.TaskGroup() as tg:
tg.create_task(t1())
tg.create_task(t2())

with self.assertRaises(BaseExceptionGroup) as cm:
await fn()

self.assertEqual(get_error_types(cm.exception), {GeneratorExit, AssertionError})

async def test_taskgroup_generator_exit_04(self):
event = asyncio.Event()
async def t():
event.set()
raise AssertionError('t failed')

async def fn():
async with asyncio.TaskGroup() as tg:
tg.create_task(t())
yield 1

g = fn()
await g.asend(None)
await event.wait() # wait for t() to run

with self.assertRaises(BaseExceptionGroup) as cm:
await g.aclose()

self.assertEqual(get_error_types(cm.exception), {GeneratorExit, AssertionError})


class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase):
loop_factory = asyncio.EventLoop
Expand Down
21 changes: 21 additions & 0 deletions Lib/test/test_dict.py
Original file line number Diff line number Diff line change
Expand Up @@ -1403,6 +1403,27 @@ def __init__(self, x, y):
self.assertEqual(list(reversed(A(1, 0).__dict__)), ['x'])
self.assertEqual(list(reversed(A(0, 1).__dict__)), ['y'])

def test_reversed_dict_after_clear_and_restore(self):
d = {}
for i in range(1000):
d[f"k{i}"] = i

for i in range(1, 1000):
del d[f"k{i}"]

iterators = (
reversed(d),
reversed(d.keys()),
reversed(d.values()),
reversed(d.items()),
)

d.clear()
d["k0"] = 0

for it in iterators:
self.assertEqual(list(it), [])

def test_dict_copy_order(self):
# bpo-34320
od = collections.OrderedDict([('a', 1), ('b', 2)])
Expand Down
14 changes: 14 additions & 0 deletions Lib/test/test_kqueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,20 @@ def test_create_queue(self):
self.assertTrue(kq.closed)
self.assertRaises(ValueError, kq.fileno)

def test_control_overflowing_timeout(self):
# gh-154836: out-of-range timeouts must raise OverflowError,
# not a (misleading) TypeError, like select(), poll() and
# epoll() do.
kq = select.kqueue()
self.addCleanup(kq.close)
for timeout in (1e300, float('inf'), 2**200):
with self.subTest(timeout=timeout):
with self.assertRaises(OverflowError):
kq.control(None, 0, timeout)
# Non-numbers still raise TypeError.
with self.assertRaises(TypeError):
kq.control(None, 0, "0.1")

def test_create_event(self):
from operator import lt, le, gt, ge

Expand Down
26 changes: 26 additions & 0 deletions Lib/test/test_subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -4300,5 +4300,31 @@ def test_fast_path_avoid_busy_loop(self):
self.assertEqual(p.wait(timeout=support.LONG_TIMEOUT), 0)
self.assertFalse(m.called)

@unittest.skipIf(mswindows, "requires the POSIX wait implementation")
def test_wait_huge_timeout(self):
# gh-154836: very large timeout values used to overflow the C
# timestamp conversion in poll() / kqueue.control() and raise
# OverflowError / TypeError.
for timeout in (10**10, sys.maxsize, float('inf')):
with self.subTest(timeout=timeout):
p = subprocess.Popen(ZERO_RETURN_CMD)
self.assertEqual(p.wait(timeout=timeout), 0)

@unittest.skipIf(mswindows, "requires the POSIX wait implementation")
def test_run_huge_timeout(self):
# gh-154836: same as test_wait_huge_timeout, via the
# subprocess.run() / communicate() code path.
cp = subprocess.run(ZERO_RETURN_CMD, timeout=1e10)
self.assertEqual(cp.returncode, 0)

@unittest.skipIf(mswindows, "requires the POSIX wait implementation")
def test_wait_slices_do_not_expire_early(self):
# A clamped wait slice must not raise TimeoutExpired before the
# real deadline: with a tiny slice limit, a process that
# outlives many slices must still be waited for successfully.
with mock.patch.object(subprocess, "_MAXIMUM_WAIT_TIMEOUT", 0.01):
p = subprocess.Popen(self.COMMAND) # sleeps 0.3s
self.assertEqual(p.wait(timeout=support.SHORT_TIMEOUT), 0)

if __name__ == "__main__":
unittest.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix an out-of-bounds access in reverse dictionary iterators when the
underlying dictionary is cleared and modified after the iterator is created.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix :meth:`subprocess.Popen.wait` raising :exc:`TypeError` (macOS and other
BSDs) or :exc:`OverflowError` (Linux) for very large *timeout* values such
as ``float('inf')``, a 3.15 regression in the new event-driven wait. Also
fix :meth:`select.kqueue.control` masking :exc:`OverflowError` for
out-of-range timeouts as :exc:`TypeError`.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix ::class:`asyncio.TaskGroup` to not wrap a :exc:`GeneratorExit` into a
:exc:`BaseExceptionGroup` if it was raised by the body of the task group and
none of the tasks in the group raised exceptions.
8 changes: 5 additions & 3 deletions Modules/selectmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -2365,9 +2365,11 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist,
else {
if (_PyTime_FromSecondsObject(&timeout,
otimeout, _PyTime_ROUND_TIMEOUT) < 0) {
PyErr_Format(PyExc_TypeError,
"timeout must be a real number or None, not %T",
otimeout);
if (PyErr_ExceptionMatches(PyExc_TypeError)) {
PyErr_Format(PyExc_TypeError,
"timeout must be a real number or None, not %T",
otimeout);
}
return NULL;
}

Expand Down
5 changes: 4 additions & 1 deletion Objects/dictobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -6276,9 +6276,12 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self)
int index = get_index_from_order(d, i);
key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key);
value = d->ma_values->values[index];
assert (value != NULL);
assert(value != NULL);
}
else {
if (i >= k->dk_nentries) {
goto fail;
}
if (DK_IS_UNICODE(k)) {
PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i];
while (entry_ptr->me_value == NULL) {
Expand Down
Loading