From adf836ebddd89793afb6d43a79d0a0739ee5514b Mon Sep 17 00:00:00 2001 From: yangbaechu <45089264+yangbaechu@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:57:00 +0900 Subject: [PATCH 1/4] gh-154936: Fix JSON control character error position (GH-154941) Report the index of the invalid literal control character instead of the following index in the pure Python decoder. --- Lib/json/decoder.py | 2 +- Lib/test/test_json/test_unicode.py | 8 +++++++- .../2026-07-30-23-29-56.gh-issue-154936.a90443.rst | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-30-23-29-56.gh-issue-154936.a90443.rst diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 364e44d40cc3073..3150a29405f75c3 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -97,7 +97,7 @@ def py_scanstring(s, end, strict=True, if strict: #msg = "Invalid control character %r at" % (terminator,) msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) + raise JSONDecodeError(msg, s, end - 1) else: _append(terminator) continue diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 1aa9546dc463061..291580cb047be46 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -44,7 +44,13 @@ def test_ascii_non_printable_decode(self): '\b\t\n\f\r') s = ''.join(map(chr, range(32))) for c in s: - self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + with self.subTest(control_character=ord(c)): + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(f'"a{c}b"') + error = cm.exception + self.assertEqual(error.pos, 2) + self.assertEqual(error.lineno, 1) + self.assertEqual(error.colno, 3) self.assertEqual(self.loads(f'"{s}"', strict=False), s) self.assertEqual(self.loads('"\x7f"'), '\x7f') diff --git a/Misc/NEWS.d/next/Library/2026-07-30-23-29-56.gh-issue-154936.a90443.rst b/Misc/NEWS.d/next/Library/2026-07-30-23-29-56.gh-issue-154936.a90443.rst new file mode 100644 index 000000000000000..19660c406eb2d0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-30-23-29-56.gh-issue-154936.a90443.rst @@ -0,0 +1,2 @@ +Fix the pure Python :mod:`json` decoder to report the correct position for +invalid literal control characters in JSON strings. From 10a84540c2c2533e31f5f0649cc4e4afdd991ba2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 31 Jul 2026 12:04:30 +0300 Subject: [PATCH 2/4] gh-154848: Enforce frame boundaries in the C unpickler (GH-154893) The C unpickler did not enforce PEP 3154 frame boundaries: an opcode or its argument could straddle a frame, and a new frame could begin before the previous one ended. Such reads now raise UnpicklingError, as in the pure Python implementation. Co-Authored-By: Claude Opus 4.8 --- Lib/test/pickletester.py | 42 +++++++++ Lib/test/test_pickle.py | 2 +- ...-07-29-18-30-00.gh-issue-154848.Fr4meB.rst | 6 ++ Modules/_pickle.c | 89 +++++++++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-18-30-00.gh-issue-154848.Fr4meB.rst diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9ba498ce8f575de..83a71b7efedd379 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1246,6 +1246,48 @@ def test_frame_readline(self): # 15: . STOP self.assertEqual(self.loads(pickled), 42) + def test_frame_ends_at_opcode_boundary(self): + # A frame may end exactly between two opcodes; the following opcodes + # are then read from outside the frame. + for pickled in [ + b'\x80\x04\x95\x01\x00\x00\x00\x00\x00\x00\x00N.', # FRAME 1, NONE, STOP + b'\x80\x04\x95\x00\x00\x00\x00\x00\x00\x00\x00N.', # empty FRAME, NONE, STOP + ]: + with self.subTest(pickled=pickled): + self.assertIsNone(self.loads(pickled)) + + def test_frame_does_not_straddle_boundary(self): + # An opcode or its argument must not cross a frame boundary + # (PEP 3154). Such a pickle must be rejected rather than silently + # reading past the declared frame length, which would make the + # meaning of the pickle diverge from its pickletools disassembly. + # See gh-154848. + for pickled in [ + # FRAME 6; UNICODE argument read by readline() straddles the frame. + b'\x80\x04\x95\x06\x00\x00\x00\x00\x00\x00\x00Vhelloworld\n.', + # FRAME 6; BINUNICODE argument straddles the frame. + b'\x80\x04\x95\x06\x00\x00\x00\x00\x00\x00\x00' + b'X\x0a\x00\x00\x00helloworld.', + # FRAME 3; SHORT_BINBYTES argument straddles the frame. + b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00C\x0ahelloworld.', + # FRAME 9; GLOBAL argument (second line) straddles the frame. + b'\x80\x04\x95\x09\x00\x00\x00\x00\x00\x00\x00cbuiltins\nprint\n.', + ]: + self.check_unpickling_error(self.truncated_errors, pickled) + + def test_nested_frame(self): + # A new frame must not begin before the current one has ended: here + # the outer frame still has data left after the inner frame header. + pickled = (b'\x80\x04\x95\x0c\x00\x00\x00\x00\x00\x00\x00' + b'N\x95\x00\x00\x00\x00\x00\x00\x00\x00NN.') + self.check_unpickling_error(self.truncated_errors, pickled) + + # But the inner frame header may lie inside the outer frame as long as + # it exactly consumes it. + pickled = (b'\x80\x04\x95\x0a\x00\x00\x00\x00\x00\x00\x00' + b'N\x95\x00\x00\x00\x00\x00\x00\x00\x00N.') + self.assertIsNone(self.loads(pickled)) + def test_compat_unpickle(self): # xrange(1, 7) pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 48375cf459ea0b1..21f45339e8a3442 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -524,7 +524,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n3P 2P2n2i5P 2P3n8P2n3i') + basesize = support.calcobjsize('2P2n3P 2P2n2i5P 2P5n8P2n3i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. diff --git a/Misc/NEWS.d/next/Library/2026-07-29-18-30-00.gh-issue-154848.Fr4meB.rst b/Misc/NEWS.d/next/Library/2026-07-29-18-30-00.gh-issue-154848.Fr4meB.rst new file mode 100644 index 000000000000000..02ca6c6346f55b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-18-30-00.gh-issue-154848.Fr4meB.rst @@ -0,0 +1,6 @@ +The :mod:`pickle` C accelerator now enforces frame boundaries when +unpickling, as the pure Python implementation already did. An argument that +straddles a frame boundary, or a frame that begins before the previous one has +ended, now raises :exc:`pickle.UnpicklingError` instead of +being silently read across the boundary. This prevents the loaded data from +diverging from the :mod:`pickletools` disassembly of the same pickle. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 034dd06b6ab4609..90200a4379319d9 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -669,6 +669,10 @@ typedef struct UnpicklerObject { Py_ssize_t input_len; Py_ssize_t next_read_idx; Py_ssize_t prefetched_idx; /* index of first prefetched byte */ + Py_ssize_t frame_end; /* End of the current frame, or -1 if not in a + frame. While in a frame, input_len is capped + here so reads can't cross it (PEP 3154). */ + Py_ssize_t saved_input_len; /* input_len to restore when the frame ends. */ PyObject *read; /* read() method of the input stream. */ PyObject *readinto; /* readinto() method of the input stream. */ @@ -1250,6 +1254,7 @@ _Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input) self->input_len = self->buffer.len; self->next_read_idx = 0; self->prefetched_idx = self->input_len; + self->frame_end = -1; return self->input_len; } @@ -1260,6 +1265,30 @@ bad_readline(PickleState *st) return -1; } +/* End the current frame, uncapping input_len. The frame must be fully read. */ +static void +_Unpickler_EndFrame(UnpicklerObject *self) +{ + assert(self->frame_end >= 0); + assert(self->next_read_idx == self->frame_end); + self->input_len = self->saved_input_len; + self->frame_end = -1; +} + +/* Reached the end of the current frame. If straddle, the read crosses the + frame boundary (PEP 3154) and fails; otherwise end the frame. */ +static int +_Unpickler_LeaveFrame(PickleState *st, UnpicklerObject *self, int straddle) +{ + if (straddle) { + PyErr_SetString(st->UnpicklingError, + "pickle exhausted before end of frame"); + return -1; + } + _Unpickler_EndFrame(self); + return 0; +} + /* Skip any consumed data that was only prefetched using peek() */ static int _Unpickler_SkipConsumed(UnpicklerObject *self) @@ -1444,6 +1473,19 @@ _Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t return -1; } + if (self->frame_end >= 0) { + if (_Unpickler_LeaveFrame(st, self, + self->next_read_idx < self->frame_end) < 0) { + return -1; + } + /* Frame ended; the read may now be satisfied from the buffer. */ + if (n <= self->input_len - self->next_read_idx) { + *s = self->input_buffer + self->next_read_idx; + self->next_read_idx += n; + return n; + } + } + /* This case is handled by the _Unpickler_Read() macro for efficiency */ assert(self->next_read_idx + n > self->input_len); @@ -1489,6 +1531,25 @@ _Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, } } + if (self->frame_end >= 0) { + /* in_buffer > 0 is next_read_idx < frame_end on entry: frame data was + consumed above, so the read crosses the frame boundary. */ + if (_Unpickler_LeaveFrame(state, self, in_buffer > 0) < 0) { + return -1; + } + in_buffer = self->input_len - self->next_read_idx; + if (in_buffer > 0) { + Py_ssize_t to_read = Py_MIN(in_buffer, n); + memcpy(buf, self->input_buffer + self->next_read_idx, to_read); + self->next_read_idx += to_read; + buf += to_read; + n -= to_read; + if (n == 0) { + return n; + } + } + } + /* Read from file */ if (!self->read) { /* We're unpickling memory, this means the input is truncated */ @@ -1547,6 +1608,7 @@ _Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) { Py_ssize_t i, num_read; +rescan: for (i = self->next_read_idx; i < self->input_len; i++) { if (self->input_buffer[i] == '\n') { char *line_start = self->input_buffer + self->next_read_idx; @@ -1555,6 +1617,14 @@ _Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) return _Unpickler_CopyLine(self, line_start, num_read, result); } } + if (self->frame_end >= 0) { + if (_Unpickler_LeaveFrame(state, self, + self->next_read_idx < self->frame_end) < 0) { + return -1; + } + /* Frame ended; continue the line past its end. */ + goto rescan; + } if (!self->read) return bad_readline(state); @@ -1729,6 +1799,8 @@ _Unpickler_New(PyObject *module) self->input_len = 0; self->next_read_idx = 0; self->prefetched_idx = 0; + self->frame_end = -1; + self->saved_input_len = 0; self->read = NULL; self->readinto = NULL; self->readline = NULL; @@ -7078,6 +7150,17 @@ load_frame(PickleState *state, UnpicklerObject *self) if (_Unpickler_Read(self, state, &s, 8) < 0) return -1; + /* A new frame must not begin before the current one ends (PEP 3154). Its + header may lie at the tail of the current frame if it drains it. */ + if (self->frame_end >= 0) { + if (self->next_read_idx < self->frame_end) { + PyErr_SetString(state->UnpicklingError, + "beginning of a new frame before end of current frame"); + return -1; + } + _Unpickler_EndFrame(self); + } + frame_len = calc_binsize(s, 8); if (frame_len < 0) { PyErr_Format(PyExc_OverflowError, @@ -7091,6 +7174,12 @@ load_frame(PickleState *state, UnpicklerObject *self) /* Rewind to start of frame */ self->next_read_idx -= frame_len; + + /* Cap input_len at the frame end so reads can't cross it (PEP 3154); + _Unpickler_EndFrame() restores it when the frame is fully read. */ + self->frame_end = self->next_read_idx + frame_len; + self->saved_input_len = self->input_len; + self->input_len = self->frame_end; return 0; } From d2ace0c42fa21c4598b7fd3a67718c5da00b66a4 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Fri, 31 Jul 2026 13:56:52 +0300 Subject: [PATCH 3/4] gh-154863: Fix iconv encoding of ISO-2022-CN-EXT returning empty bytes (GH-154899) The flush that emits the pending shift sequence can report a nonreversible conversion, which the loop mistook for a substituted character. It then retried while still flushing, converted nothing and returned the empty output buffer. --- Lib/test/test_codecs.py | 31 +++++++++++++++++++++++++++++++ Objects/unicodeobject.c | 6 +++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 31704955df3e14e..f6f8089a065059b 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3679,6 +3679,10 @@ def iconv_encoding_available(name): # Encodings iconv may provide but for which CPython has no built-in codec # (cp1047 is EBCDIC, i.e. not ASCII-compatible). _ICONV_ONLY = ['cp1047', 'cp1133', 'GEORGIAN-PS', 'ARMSCII-8'] +# Encodings that leave a shift state pending, so encoding ends with a flush. +_ICONV_SHIFT_STATE = [('ISO-2022-CN-EXT', 'ABC\u4e2dDEF'), + ('ISO-2022-CN', 'ABC\u4e2dDEF'), + ('ISO-2022-JP', 'ABC\u65e5DEF')] @unittest.skipUnless(hasattr(codecs, 'iconv_encode'), @@ -3812,6 +3816,33 @@ def test_encode_kinds(self): with self.subTest(text=text): self.assertEqual(text.encode('iconv:' + enc), text.encode(enc)) + def test_encode_shift_state_flush(self): + # Encoding ends with a flush that emits the pending shift sequence. Its + # return value counts nonreversible conversions, and some iconv + # implementations make it positive for the flush itself (glibc does for + # ISO-2022-CN-EXT). That must not be read as a substituted character: + # doing so discarded the whole output, ASCII included. + # + # Only the ASCII around the character is checked, not a full round-trip. + # An iconv that cannot represent the character either rejects it or + # substitutes for it silently, as macOS does for ISO-2022-CN. + tested = False + for enc, text in _ICONV_SHIFT_STATE: + if not iconv_encoding_available(enc): + continue + with self.subTest(encoding=enc): + try: + data = codecs.iconv_encode(enc, text)[0] + except UnicodeEncodeError: + continue + tested = True + self.assertNotEqual(data, b'') + decoded = codecs.iconv_decode(enc, data, 'strict', True)[0] + self.assertStartsWith(decoded, 'ABC') + self.assertEndsWith(decoded, 'DEF') + if not tested: + self.skipTest('no shift-state iconv encoding is available') + def test_encode_surrogateescape(self): # A lone surrogate lives in the 2-byte kind and round-trips. enc = self.require('ASCII') diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a4550c0f5f3363a..eec02f662e79059 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8462,6 +8462,9 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, } if (ret != (size_t)-1) { + if (flushing) { + break; + } /* A positive result counts nonreversible conversions: iconv() substituted an unencodable character instead of failing with EILSEQ (musl and *BSD citrus do this). Treat it as unencodable @@ -8479,9 +8482,6 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, out = out_before; up -= unit; } - else if (flushing) { - break; - } else if (careful && up < uend) { continue; } From 7d760134cc20f444412c54fc8395ca6f2421ad73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Fri, 31 Jul 2026 18:58:59 +0800 Subject: [PATCH 4/4] gh-154738: Propagate reparse-deferral setting to pyexpat subparsers (GH-154739) --- Lib/test/test_pyexpat.py | 7 +++++++ .../Library/2026-07-26-00-00-00.gh-issue-154738.Rd5Pxe.rst | 3 +++ Modules/pyexpat.c | 1 + 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-07-26-00-00-00.gh-issue-154738.Rd5Pxe.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index a2ef67baadd4afb..54dfa95ce8bff17 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1045,6 +1045,13 @@ def test_parent_parser_outlives_its_subparsers__chain(self): del parser del subparser + def test_subparser_inherits_reparse_deferral(self): + for enabled in (True, False): + parser = expat.ParserCreate() + parser.SetReparseDeferralEnabled(enabled) + subparser = parser.ExternalEntityParserCreate(None) + self.assertEqual(subparser.GetReparseDeferralEnabled(), enabled) + class ExternalEntityParserCreateErrorTest(unittest.TestCase): """ExternalEntityParserCreate error paths should not crash or leak diff --git a/Misc/NEWS.d/next/Library/2026-07-26-00-00-00.gh-issue-154738.Rd5Pxe.rst b/Misc/NEWS.d/next/Library/2026-07-26-00-00-00.gh-issue-154738.Rd5Pxe.rst new file mode 100644 index 000000000000000..1a6e1a301db466f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-26-00-00-00.gh-issue-154738.Rd5Pxe.rst @@ -0,0 +1,3 @@ +Fix :meth:`!ExternalEntityParserCreate` not propagating the reparse-deferral +setting to the subparser, which left :meth:`!GetReparseDeferralEnabled` +returning an uninitialized value. Patch by tonghuaroot. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f9e7e057f7e0f7b..397a441f574fe45 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1118,6 +1118,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->specified_attributes = self->specified_attributes; new_parser->in_callback = 0; new_parser->ns_prefixes = self->ns_prefixes; + new_parser->reparse_deferral_enabled = self->reparse_deferral_enabled; new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context, encoding); // The new subparser will make use of the parent XML_Parser inside of Expat.