Merge branch 'main' into fix-proactor-error-hang

Author: graingert

Lib/asyncio/proactor_events.py

@@ -778,8 +778,8 @@ async def _sock_sendfile_native(self, sock, file, offset, count):
     async def _sendfile_native(self, transp, file, offset, count):
         resume_reading = transp.is_reading()
         transp.pause_reading()
-        await transp._make_empty_waiter()
         try:
+            await transp._make_empty_waiter()
             return await self.sock_sendfile(transp._sock, file, offset, count,
                                             fallback=False)
         finally:

Lib/asyncio/selector_events.py

@@ -739,8 +739,8 @@ async def _sendfile_native(self, transp, file, offset, count):
         del self._transports[transp._sock_fd]
         resume_reading = transp.is_reading()
         transp.pause_reading()
-        await transp._make_empty_waiter()
         try:
+            await transp._make_empty_waiter()
             return await self.sock_sendfile(transp._sock, file, offset, count,
                                             fallback=False)
         finally:

Lib/difflib.py

@@ -31,7 +31,7 @@
            'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match']
 
 from heapq import nlargest as _nlargest
-from collections import namedtuple as _namedtuple
+from collections import deque as _deque, namedtuple as _namedtuple
 from types import GenericAlias
 lazy from _colorize import can_colorize, get_theme
 
@@ -1571,7 +1571,7 @@ def _line_pair_iterator():
         is defined) does not need to be of module scope.
         """
         line_iterator = _line_iterator()
-        fromlines,tolines=[],[]
+        fromlines, tolines = _deque(), _deque()
         while True:
             # Collecting lines of text until we have a from/to pair
             while (len(fromlines)==0 or len(tolines)==0):
@@ -1584,8 +1584,8 @@ def _line_pair_iterator():
                 if to_line is not None:
                     tolines.append((to_line,found_diff))
             # Once we have a pair, remove them from the collection and yield it
-            from_line, fromDiff = fromlines.pop(0)
-            to_line, to_diff = tolines.pop(0)
+            from_line, fromDiff = fromlines.popleft()
+            to_line, to_diff = tolines.popleft()
             yield (from_line,to_line,fromDiff or to_diff)
 
     # Handle case where user does not want context differencing, just yield

Lib/idlelib/idle_test/test_configdialog.py

@@ -150,8 +150,8 @@ def test_fontlist_key(self):
         font = d.fontlist.get('active')
 
         # Test Down key.
-        fontlist.focus_force()
         fontlist.update()
+        fontlist.focus_force()
         fontlist.event_generate('<Key-Down>')
         fontlist.event_generate('<KeyRelease-Down>')
 
@@ -160,8 +160,8 @@ def test_fontlist_key(self):
         self.assertIn(d.font_name.get(), down_font.lower())
 
         # Test Up key.
-        fontlist.focus_force()
         fontlist.update()
+        fontlist.focus_force()
         fontlist.event_generate('<Key-Up>')
         fontlist.event_generate('<KeyRelease-Up>')
 

Lib/test/test_asyncio/test_sendfile.py

@@ -376,6 +376,47 @@ def test_sendfile(self):
         self.assertEqual(srv_proto.data, self.DATA)
         self.assertEqual(self.file.tell(), len(self.DATA))
 
+    def test_sendfile_cancel_empty_waiter(self):
+        for reading in (True, False):
+            with self.subTest(reading=reading):
+                srv_proto, cli_proto = self.prepare_sendfile()
+                transport = cli_proto.transport
+                if not reading:
+                    transport.pause_reading()
+                waiter = self.loop.create_future()
+
+                def make_empty_waiter():
+                    transport._empty_waiter = waiter
+                    return waiter
+
+                with mock.patch.object(transport, '_make_empty_waiter',
+                                       side_effect=make_empty_waiter):
+                    task = self.loop.create_task(
+                        self.loop.sendfile(transport, self.file))
+                    test_utils.run_briefly(self.loop)
+                    self.assertIs(transport._empty_waiter, waiter)
+                    self.assertFalse(waiter.done())
+                    self.assertFalse(transport.is_reading())
+                    task.cancel()
+                    with self.assertRaises(asyncio.CancelledError):
+                        self.run_loop(task)
+
+                try:
+                    self.assertIsNone(transport._empty_waiter)
+                    self.assertEqual(transport.is_reading(), reading)
+                    if isinstance(self.loop, asyncio.SelectorEventLoop):
+                        self.assertIs(
+                            self.loop._transports[transport._sock_fd],
+                            transport)
+                finally:
+                    transport._reset_empty_waiter()
+
+                ret = self.run_loop(self.loop.sendfile(transport, self.file))
+                transport.close()
+                self.run_loop(srv_proto.done)
+                self.assertEqual(ret, len(self.DATA))
+                self.assertEqual(srv_proto.data, self.DATA)
+
     def test_sendfile_force_fallback(self):
         srv_proto, cli_proto = self.prepare_sendfile()
 

Lib/test/test_capi/test_tokenizer.py

@@ -9,6 +9,9 @@ class TokenizerTests(unittest.TestCase):
     def test_source(self):
         _testinternalcapi.test_tokenizer_source()
 
+    def test_source_discard(self):
+        _testinternalcapi.test_tokenizer_source_discard()
+
     def test_cursor(self):
         _testinternalcapi.test_tokenizer_cursor()
 

Lib/test/test_difflib.py

@@ -200,6 +200,26 @@ def test_mdiff_catch_stop_iteration(self):
             [((1, '\x00-2\x01'), (1, '\x00+3\x01'), True)],
         )
 
+    def test_mdiff_lopsided_replace(self):
+        self.assertEqual(
+            list(difflib._mdiff(["a\n"] * 4, ["b\n"])),
+            [
+                ((1, '\x00-a\n\x01'), (1, '\x00+b\n\x01'), True),
+                ((2, '\x00-a\n\x01'), ('', '\n'), True),
+                ((3, '\x00-a\n\x01'), ('', '\n'), True),
+                ((4, '\x00-a\n\x01'), ('', '\n'), True),
+            ],
+        )
+        self.assertEqual(
+            list(difflib._mdiff(["a\n"], ["b\n"] * 4)),
+            [
+                ((1, '\x00-a\n\x01'), (1, '\x00+b\n\x01'), True),
+                (('', '\n'), (2, '\x00+b\n\x01'), True),
+                (('', '\n'), (3, '\x00+b\n\x01'), True),
+                (('', '\n'), (4, '\x00+b\n\x01'), True),
+            ],
+        )
+
 
 patch914575_from1 = """
    1. Beautiful is beTTer than ugly.

Lib/test/test_hashlib.py

@@ -18,6 +18,8 @@
 import tempfile
 import threading
 import unittest
+from functools import partial
+from operator import attrgetter
 from test import support
 from test.support import _4G, bigmemtest
 from test.support import hashlib_helper
@@ -52,18 +54,39 @@
     def get_fips_mode():
         return 0
 
+
+try:
+    import _md5
+except ImportError:
+    _md5 = None
+requires_md5 = unittest.skipUnless(_md5, 'requires _md5')
+
+
 try:
     import _blake2
 except ImportError:
     _blake2 = None
-
 requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')
 
+
+try:
+    import _sha1
+except ImportError:
+    _sha1 = None
+requires_sha1 = unittest.skipUnless(_sha1, 'requires _sha1')
+
+
+try:
+    import _sha2
+except ImportError:
+    _sha2 = None
+requires_sha2 = unittest.skipUnless(_sha2, 'requires _sha2')
+
+
 try:
     import _sha3
 except ImportError:
     _sha3 = None
-
 requires_sha3 = unittest.skipUnless(_sha3, 'requires _sha3')
 
 
@@ -1418,5 +1441,76 @@ def scrypt(password=b"password", /, **kwargs):
         self.assertRaises(numeric_exc_types, scrypt, dklen=MAX_DKLEN + 1)
 
 
+@threading_helper.requires_working_threading()
+class TestTSAN(unittest.TestCase):
+
+    @threading_helper.reap_threads
+    def check_attribute(self, write, read, expected, nthreads=8):
+        ready = threading.Event()
+        barrier = threading.Barrier(nthreads)
+
+        def writer():
+            barrier.wait()
+            while not ready.is_set():
+                write()
+
+        def reader():
+            barrier.wait()
+            while not ready.is_set():
+                self.assertEqual(read(), expected)
+
+        targets = [writer if i % 2 else reader for i in range(nthreads)]
+        workers = [threading.Thread(target=target) for target in targets]
+        with threading_helper.start_threads(workers, unlock=ready.set):
+            pass
+
+    def check_HACL_attribute(self, module, version, attrname):
+        blob = b"A" * 65536
+        obj = getattr(module, version)()
+        update = partial(obj.update, blob)
+        read = attrgetter(attrname)
+        self.check_attribute(update, partial(read, obj), read(obj))
+
+    @requires_md5
+    @support.subTests("attrname", ["block_size", "digest_size"])
+    def test_HACL_md5_attributes(self, attrname):
+        self.check_HACL_attribute(_md5, "md5", attrname)
+
+    @requires_sha1
+    @support.subTests("attrname", ["block_size", "digest_size"])
+    def test_HACL_sha1_attributes(self, attrname):
+        self.check_HACL_attribute(_sha1, "sha1", attrname)
+
+    @requires_sha2
+    @support.subTests("size", [224, 256, 384, 512])
+    @support.subTests("attrname", ["block_size", "digest_size"])
+    def test_HACL_sha2_attributes(self, size, attrname):
+        self.check_HACL_attribute(_sha2, f"sha{size}", attrname)
+
+    @requires_sha3
+    @support.subTests("size", [224, 256, 384, 512])
+    @support.subTests(
+        "attrname",
+        ["block_size", "digest_size", "_capacity_bits", "_rate_bits"],
+    )
+    def test_HACL_sha3_attributes(self, size, attrname):
+        self.check_HACL_attribute(_sha3, f"sha3_{size}", attrname)
+
+    @requires_sha3
+    @support.subTests("size", [128, 256])
+    @support.subTests(
+        "attrname",
+        ["block_size", "digest_size", "_capacity_bits", "_rate_bits"],
+    )
+    def test_HACL_shake_attributes(self, size, attrname):
+        self.check_HACL_attribute(_sha3, f"shake_{size}", attrname)
+
+    @requires_blake2
+    @support.subTests("version", ["blake2s", "blake2b"])
+    @support.subTests("attrname", ["block_size", "digest_size"])
+    def test_HACL_blake2_attributes(self, version, attrname):
+        self.check_HACL_attribute(_blake2, version, attrname)
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_io/test_memoryio.py

@@ -1025,6 +1025,18 @@ def test_cow_mutable(self):
         memio = self.ioclass(ba)
         self.assertEqual(sys.getrefcount(ba), old_rc)
 
+    def test_write_with_export(self):
+        memio = self.ioclass(b"abcd")
+        memio.seek(2)
+        with memio.getbuffer() as view:
+            self.assertRaises(BufferError, memio.__init__, b"replacement")
+            self.assertEqual(memio.tell(), 2)
+            self.assertEqual(memio.getvalue(), b"abcd")
+            self.assertEqual(bytes(view), b"abcd")
+        memio.write(b"X")
+        self.assertEqual(memio.getvalue(), b"abXd")
+
+
 class CStringIOTest(PyStringIOTest):
     ioclass = io.StringIO
     UnsupportedOperation = io.UnsupportedOperation

Lib/test/test_repl.py

@@ -198,6 +198,22 @@ def test_lexer_buffer_realloc_with_null_start(self):
         self.assertEqual(p.returncode, 0)
         self.assertIn(long_value, output)
 
+    @cpython_only
+    def test_multiline_fstring_source_reallocation(self):
+        long_line = " " * 9000 + "+ 2"
+        user_input = (
+            'value = f"""{(\n'
+            '1\n'
+            f'{long_line}\n'
+            ')}"""\n'
+            'print(value)\n'
+        )
+        p = spawn_repl()
+        p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+        self.assertIn(">>> 3\n>>> ", output)
+
     def test_close_stdin(self):
         user_input = dedent('''
             import os