[pull] main from python:main

Include/internal/pycore_ceval.h

@@ -131,7 +131,7 @@ _Py_CODEUNIT *_PyTier2Interpreter(
 );
 #endif
 
-extern _PyJitEntryFuncPtr _Py_jit_entry;
+PyAPI_DATA(_PyJitEntryFuncPtr) _Py_jit_entry;
 
 extern PyObject*
 _PyEval_Vector(PyThreadState *tstate,

Include/internal/pycore_global_objects_fini.h

@@ -0,0 +1,94 @@
+#ifndef Py_INTERNAL_GLOBAL_OBJECTS_FINI_H
+#define Py_INTERNAL_GLOBAL_OBJECTS_FINI_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_DEBUG
+
+#include "pycore_bytesobject.h"   // _PyBytes_CheckOverflow()
+#include "pycore_long.h"          // TAG_FROM_SIGN_AND_SIZE()
+
+static inline void
+_PyStaticObject_CheckSingleton(PyObject *obj, PyTypeObject *type)
+{
+    // Check PyObject.ob_refcnt
+    _PyObject_ASSERT(obj, _Py_IsImmortal(obj));
+
+    // Check PyObject.ob_type
+    _PyObject_ASSERT(obj, Py_TYPE(obj) == type);
+}
+
+
+static void
+_PyStaticObject_CheckLongSingleton(PyObject *obj, long value, int is_bool)
+{
+    PyTypeObject *type = is_bool ? &PyBool_Type : &PyLong_Type;
+    _PyStaticObject_CheckSingleton(obj, type);
+
+    // Check _PyLong_CompactValue()
+    Py_ssize_t compact = _PyLong_CompactValue((const PyLongObject *)obj);
+    _PyObject_ASSERT(obj, compact == value);
+
+    // Check tv_tag and ob_digit[0]
+    _PyLongValue *long_value = &((PyLongObject*)obj)->long_value;
+    int sign = (value == 0) ? 0 : ((value < 0) ? -1 : 1);
+    uintptr_t lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (value == 0) ? 0 : 1);
+    if (!is_bool) {
+        lv_tag |= IMMORTALITY_BIT_MASK;
+    }
+    _PyObject_ASSERT(obj, long_value->lv_tag == lv_tag);
+    _PyObject_ASSERT(obj, long_value->ob_digit[0] == Py_ABS(value));
+}
+
+
+static inline void
+_PyStaticObject_CheckBytesSingleton(PyObject *obj,
+                                    Py_ssize_t size, unsigned char ch)
+{
+    _PyStaticObject_CheckSingleton(obj, &PyBytes_Type);
+    _PyObject_ASSERT(obj, PyBytes_GET_SIZE(obj) == size);
+    const unsigned char *str = (const unsigned char *)PyBytes_AS_STRING(obj);
+    _PyObject_ASSERT(obj, str[0] == ch);
+    _PyBytes_CheckOverflow(obj, obj, "bytes singleton");
+}
+
+static void
+_PyStaticObject_CheckUnicode(PyObject *obj, const char *str, Py_ssize_t length)
+{
+    _PyStaticObject_CheckSingleton(obj, &PyUnicode_Type);
+    _PyObject_ASSERT(obj, _PyUnicode_CheckConsistency(obj, 1));
+    _PyObject_ASSERT(obj, PyUnicode_GET_LENGTH(obj) == length);
+    _PyObject_ASSERT(obj, PyUnicode_KIND(obj) == PyUnicode_1BYTE_KIND);
+    const Py_UCS1 *data = PyUnicode_1BYTE_DATA(obj);
+    _PyObject_ASSERT(obj, memcmp(data, str, length) == 0);
+    _PyObject_ASSERT(obj, data[length] == 0);
+}
+
+
+static void
+_PyStaticObject_CheckUnicodeCharSingleton(PyObject *obj, unsigned char ch)
+{
+    _PyStaticObject_CheckUnicode(obj, (char *)&ch, 1);
+    _PyObject_ASSERT(obj, PyUnicode_IS_ASCII(obj) == (ch <= 127));
+}
+
+
+static void
+_PyStaticObject_CheckUnicodeSingleton(PyObject *obj,
+                                      const char *str, Py_ssize_t length)
+{
+    _PyStaticObject_CheckUnicode(obj, str, length);
+    _PyObject_ASSERT(obj, PyUnicode_IS_ASCII(obj));
+}
+
+#endif  // Py_DEBUG
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_GLOBAL_OBJECTS_FINI_H */

Lib/pprint.py

@@ -187,14 +187,19 @@ def isreadable(self, object):
         return readable and not recursive
 
     def _format(self, object, stream, indent, allowance, context, level):
+        # Width of any "key: " prefix already written on the current line by
+        # _format_child().  In expand mode `indent` is the block indent and so
+        # does not include it, but it still consumes width here.
+        prefix_len = self._pending_prefix_len
+        self._pending_prefix_len = 0
         objid = id(object)
         if objid in context:
             stream.write(_recursion(object))
             self._recursive = True
             self._readable = False
             return
         rep = self._repr(object, context, level)
-        max_width = self._width - indent - allowance
+        max_width = self._width - indent - prefix_len - allowance
         if len(rep) > max_width:
             p = self._dispatch.get(type(object).__repr__, None)
             # Lazy import to improve module import time
@@ -232,6 +237,22 @@ def _child_indent(self, indent, prefix_len):
             return indent
         return indent + prefix_len
 
+    # Set by _format_child() immediately before it calls _format(), and
+    # consumed there.  Passing it out of band keeps _format()'s signature
+    # unchanged for third-party subclasses that override it.
+    _pending_prefix_len = 0
+
+    def _format_child(self, object, stream, indent, allowance, context, level,
+                      prefix_len):
+        if self._expand:
+            # Aligned mode folds the prefix into the indent (see
+            # _child_indent), so only expand mode needs to report it.
+            self._pending_prefix_len = prefix_len
+        try:
+            self._format(object, stream, indent, allowance, context, level)
+        finally:
+            self._pending_prefix_len = 0
+
     def _write_indent_padding(self, write):
         if self._expand:
             if self._indent_per_level > 0:
@@ -303,13 +324,14 @@ def _pprint_ordered_dict(self, object, stream, indent, allowance, context, level
             return
         cls = object.__class__
         stream.write(cls.__name__ + '(')
-        self._format(
+        self._format_child(
             list(object.items()),
             stream,
             self._child_indent(indent, len(cls.__name__) + 1),
             allowance + 1,
             context,
             level,
+            len(cls.__name__) + 1,
         )
         stream.write(')')
 
@@ -498,13 +520,14 @@ def _pprint_bytearray(self, object, stream, indent, allowance, context, level):
 
     def _pprint_mappingproxy(self, object, stream, indent, allowance, context, level):
         stream.write('mappingproxy(')
-        self._format(
+        self._format_child(
             object.copy(),
             stream,
             self._child_indent(indent, 13),
             allowance + 1,
             context,
             level,
+            13,
         )
         stream.write(')')
 
@@ -540,13 +563,14 @@ def _format_dict_items(self, items, stream, indent, allowance, context,
             rep = self._repr(key, context, level)
             write(rep)
             write(': ')
-            self._format(
+            self._format_child(
                 ent,
                 stream,
                 self._child_indent(indent, len(rep) + 2),
                 allowance if last else 1,
                 context,
                 level,
+                len(rep) + 2,
             )
             if not last:
                 write(delimnl)
@@ -566,13 +590,14 @@ def _format_namespace_items(self, items, stream, indent, allowance, context, lev
                 # recursive dataclass repr.
                 write("...")
             else:
-                self._format(
+                self._format_child(
                     ent,
                     stream,
                     self._child_indent(indent, len(key) + 1),
                     allowance if last else 1,
                     context,
                     level,
+                    len(key) + 1,
                 )
             if not last:
                 write(delimnl)

Lib/test/libregrtest/refleak.py

@@ -114,12 +114,15 @@ def runtest_refleak(test_name, test_func,
     rc_deltas = array('q', [0]) * repcount
     alloc_deltas = array('q', [0]) * repcount
     fd_deltas = array('q', [0]) * repcount
+    handle_deltas = array('q', [0]) * repcount
     getallocatedblocks = sys.getallocatedblocks
     gettotalrefcount = sys.gettotalrefcount
     getunicodeinternedsize = sys.getunicodeinternedsize
     fd_count = os_helper.fd_count
+    handle_count = os_helper.handle_count
     # initialize variables to make pyflakes quiet
     rc_before = alloc_before = fd_before = interned_immortal_before = 0
+    handle_before = 0
 
     if not quiet:
         print("beginning", repcount, "repetitions. Showing number of leaks "
@@ -154,13 +157,17 @@ def runtest_refleak(test_name, test_func,
         alloc_after = getallocatedblocks() - interned_immortal_after
         rc_after = gettotalrefcount()
         fd_after = fd_count()
+        handle_after = handle_count()
 
         rc_deltas[i] = rc_after - rc_before
         alloc_deltas[i] = alloc_after - alloc_before
         fd_deltas[i] = fd_after - fd_before
+        handle_deltas[i] = handle_after - handle_before
 
         if not quiet:
-            total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i])
+            # use max, not sum, so total_leaks is one of the pooled ints
+            total_leaks = max(rc_deltas[i], alloc_deltas[i],
+                              fd_deltas[i], handle_deltas[i])
             if total_leaks <= 0:
                 symbol = '.'
             elif total_leaks < 10:
@@ -178,18 +185,29 @@ def runtest_refleak(test_name, test_func,
         alloc_before = alloc_after
         rc_before = rc_after
         fd_before = fd_after
+        handle_before = handle_after
         interned_immortal_before = interned_immortal_after
 
         restore_support_xml(xml_filename)
 
     if not quiet:
         print(file=sys.stderr)
 
+    if ('multiprocessing' in test_name
+        or 'concurrent_futures' in test_name):
+        # gh-154208: Disable check for Windows handle leaks when
+        # multiprocessing is used. There is a known race condition in
+        # multiprocessing causing handle leak. Disable the multiprocessing
+        # tests to be able to check for leaks for all other tests.
+        for i in range(len(handle_deltas)):
+            handle_deltas[i] = 0
+
     failed = False
     for raw_deltas, item_name in [
         (rc_deltas, 'references'),
         (alloc_deltas, 'memory blocks'),
-        (fd_deltas, 'file descriptors')
+        (fd_deltas, 'file descriptors'),
+        (handle_deltas, 'handles'),
     ]:
         # Ignore warmup runs; convert to a list for reporting
         deltas = list(raw_deltas[warmups:])

Lib/test/support/__init__.py

@@ -1373,11 +1373,7 @@ def internal(*args, **kwargs):
         import_module('_testcapi')
         return test(*args, **kwargs)
 
-    use_tsan = check_sanitizer(thread=True)
-    reason ='not working with thread sanitizer (gh-157415)'
-    skip_if_tsan = unittest.skipIf(use_tsan, reason)
-
-    return cpython_only(skip_if_tsan(internal))
+    return cpython_only(internal)
 
 def bigaddrspacetest(f):
     """Decorator for tests that fill the address space."""

Lib/test/test_capi/test_misc.py

@@ -3094,5 +3094,51 @@ def test_ceval_decref(self):
         self.assertEqual(lines.count("DESTROY list"), 2)
 
 
+@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
+class TestCheckSingleton(unittest.TestCase):
+    # Test that _PyStaticObjects_CheckAll() detects memory corruptions in
+    # singleton objects at Python exit.
+
+    def check(self, code):
+        code = f"""if 1:
+            import _testcapi
+            from test import support
+            support.SuppressCrashReport().__enter__()
+            {code}
+        """
+        proc = assert_python_failure("-c", code)
+        return proc.err
+
+    def test_corrupt_bytes(self):
+        stderr = self.check("_testcapi.corrupt_bytes(b'a', b'#')")
+
+        self.assertIn((b'_PyStaticObject_CheckBytesSingleton: '
+                       b'Assertion "str[0] == ch" failed'), stderr)
+        self.assertIn(b"object repr     : b'#'", stderr)
+
+    def test_corrupt_unicode(self):
+        stderr = self.check("_testcapi.corrupt_unicode('a', '#')")
+
+        self.assertIn((b'_PyStaticObject_CheckUnicode: '
+                       b'Assertion "memcmp(data, str, length) == 0" failed'), stderr)
+        self.assertIn(b"object repr     : '#'", stderr)
+
+    def test_corrupt_bool(self):
+        stderr = self.check("_testcapi.corrupt_long(True, 0)")
+
+        self.assertIn((b'_PyStaticObject_CheckLongSingleton: '
+                       b'Assertion "compact == value" failed'),
+                      stderr)
+        self.assertIn(b"object repr     : True", stderr)
+
+    def test_corrupt_long(self):
+        stderr = self.check("_testcapi.corrupt_long(5, 42)")
+
+        self.assertIn((b'_PyStaticObject_CheckLongSingleton: '
+                       b'Assertion "compact == value" failed'),
+                      stderr)
+        self.assertIn(b"object repr     : 42", stderr)
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_pprint.py

@@ -1646,6 +1646,26 @@ def test_expand_dict(self):
     'corge': 7,
 }""")
 
+    def test_expand_respects_width_with_long_keys(self):
+        # gh-155596: in expand mode the width of the "key: " prefix was not
+        # counted when deciding whether a value fits on the current line, so
+        # values under long keys could overflow width.
+        obj = {'a' * 12: 1, 'b' * 20: 2, 'c' * 30: {'d' * 5: 3, 'e' * 40: 3}}
+        result = pprint.pformat(obj, expand=True)
+        self.assertEqual(result,
+"""\
+{
+ 'aaaaaaaaaaaa': 1,
+ 'bbbbbbbbbbbbbbbbbbbb': 2,
+ 'cccccccccccccccccccccccccccccc': {
+  'ddddd': 3,
+  'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee': 3,
+ },
+}""")
+        # The nested value must be broken up rather than overflowing.
+        self.assertTrue(all(len(line) <= 80 for line in result.splitlines()),
+                        max(result.splitlines(), key=len))
+
     def test_expand_ordered_dict(self):
         dummy_ordered_dict = collections.OrderedDict(
             [
@@ -1895,7 +1915,11 @@ def test_expand_chainmap(self):
             'baz': 123,
             'corge': 7,
             'foo': 'bar',
-            'quux': ['foo', 'bar', 'baz'],
+            'quux': [
+                'foo',
+                'bar',
+                'baz',
+            ],
             'qux': {
                 'baz': 123,
                 'foo': 'bar',
@@ -1939,7 +1963,10 @@ def test_expand_deque(self):
         'corge': 7,
         'foo': 'bar',
         'quux': ['foo', 'bar', 'baz'],
-        'qux': {'baz': 123, 'foo': 'bar'},
+        'qux': {
+            'baz': 123,
+            'foo': 'bar',
+        },
     },
     'foo',
     'bar',