From 56ee379175cc786420f17d4e0788f77bd34a1d1a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 25 Jul 2026 11:33:10 +0300 Subject: [PATCH 1/4] gh-154682: Fix corruption of locale-encoded text on FreeBSD, NetBSD, DragonFly BSD and macOS On these platforms wchar_t values are not Unicode code points in non-UTF-8 locales, so mbstowcs() and wcstombs() cannot be used to convert locale-encoded text, neither can wcsftime() output be used, and wcscoll(), wcsxfrm() and the wide character curses API cannot be used with Unicode wchar_t. Use iconv() in _Py_DecodeLocaleEx() and _Py_EncodeLocaleEx(), use strftime() instead of wcsftime() in time.strftime(), use strcoll() and strxfrm() on the locale-encoded strings in the locale module, and convert between Unicode and the native wchar_t form on the curses library boundary. Co-Authored-By: Claude Fable 5 --- Include/internal/pycore_fileutils.h | 22 ++ Lib/test/test_locale.py | 47 +++ Lib/test/test_readline.py | 4 + Lib/test/test_time.py | 36 ++ ...-07-25-13-50-00.gh-issue-154682.wChBSD.rst | 10 + Modules/_cursesmodule.c | 78 ++++ Modules/_localemodule.c | 59 +++ Modules/timemodule.c | 9 + Python/fileutils.c | 372 ++++++++++++++++++ 9 files changed, 637 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2c6d6daa01994ee..31807cfb98ca05a 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -225,6 +225,28 @@ extern int _Py_GetForceASCII(void); encoding. */ extern void _Py_ResetForceASCII(void); +#ifdef HAVE_LANGINFO_H +# include // CODESET +#endif + +/* On FreeBSD, NetBSD, DragonFly BSD and macOS, wchar_t values are not + Unicode code points in non-UTF-8 locales. Locale-encoded text is converted + with iconv() (see Python/fileutils.c), and the wide character + functions which use the locale (such as wcscoll()) cannot be used + with Unicode wchar_t. */ +#if (defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__)) \ + && defined(HAVE_ICONV) \ + && defined(HAVE_LANGINFO_H) && defined(CODESET) && SIZEOF_WCHAR_T == 4 +# define _Py_NON_UNICODE_WCHAR_T + +// Convert between Unicode code points and the native wchar_t form for +// C library functions which use the latter (such as the curses library). +// Export for '_curses'. +PyAPI_FUNC(int) _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size); +PyAPI_FUNC(void) _Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size); +#endif + extern int _Py_GetLocaleconvNumeric( struct lconv *lc, diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 8057c35f540841b..f51879bd23f596d 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -350,6 +350,53 @@ def test_strxfrm(self): self.assertRaises(ValueError, locale.strxfrm, 'a\0') +class TestCollationAcrossEncodings(unittest.TestCase): + # gh-154682: collation must give the same results in locales which + # only differ in the encoding. + + @support.thread_unsafe('setlocale is not thread-safe') + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") + @unittest.skipIf(sys.platform.startswith("dragonfly") or + sys.platform == "darwin", + "no collation data for non-UTF-8 locales") + def test_across_encodings(self): + def sign(x): + return (x > 0) - (x < 0) + cases = [ + ('uk_UA.UTF-8', 'uk_UA.KOI8-U', 'гґдиіїя'), + ('uk_UA.UTF-8', 'uk_UA.CP1251', 'гґдиіїя'), + ('el_GR.UTF-8', 'el_GR.ISO8859-7', 'αβδω'), + ('de_DE.UTF-8', 'de_DE.ISO8859-1', 'aäbößz'), + ('tr_TR.UTF-8', 'tr_TR.ISO8859-9', 'cçdıisşz'), + ('ca_ES.UTF-8', 'ca_ES.ISO8859-1', 'cçde'), + ] + oldloc = locale.setlocale(locale.LC_ALL) + self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc) + tested = False + for utf8_loc, legacy_loc, chars in cases: + try: + locale.setlocale(locale.LC_ALL, utf8_loc) + except locale.Error: + continue + expected_order = sorted(chars, key=locale.strxfrm) + expected_signs = [[sign(locale.strcoll(a, b)) for b in chars] + for a in chars] + try: + locale.setlocale(locale.LC_ALL, legacy_loc) + except locale.Error: + continue + with self.subTest(locale=legacy_loc): + self.assertEqual(sorted(chars, key=locale.strxfrm), + expected_order) + signs = [[sign(locale.strcoll(a, b)) for b in chars] + for a in chars] + self.assertEqual(signs, expected_signs) + tested = True + if not tested: + self.skipTest('no suitable locales') + + class TestEnUSCollation(BaseLocalizedTest, TestCollation): # Test string collation functions with a real English locale diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 11ec57a259a9abf..70b6190dd2a4460 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -125,6 +125,10 @@ def test_write_read_append(self): readline.append_history_file(-2147483648, hfilename) def test_nonascii_history(self): + try: + "entrée".encode(locale.getencoding()) + except UnicodeEncodeError as err: + self.skipTest("Locale cannot encode test data: " + format(err)) readline.clear_history() try: readline.add_history("entrée 1") diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 985984b55123ce6..f36c6a4c8a264a8 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -3,6 +3,7 @@ import decimal import enum import fractions +import locale import math import platform import sys @@ -232,6 +233,41 @@ def test_strftime(self): self.assertRaises(TypeError, time.strftime, b'%S', tt) + @support.thread_unsafe('setlocale is not thread-safe') + def test_strftime_locale_encodings(self): + # gh-154682: the result must be the same in locales which only + # differ in the encoding. + pairs = [ + ('uk_UA.UTF-8', 'uk_UA.KOI8-U'), + ('uk_UA.UTF-8', 'uk_UA.CP1251'), + ('el_GR.UTF-8', 'el_GR.ISO8859-7'), + ('ja_JP.UTF-8', 'ja_JP.eucJP'), + ('de_DE.UTF-8', 'de_DE.ISO8859-1'), + ('tr_TR.UTF-8', 'tr_TR.ISO8859-9'), + ('ca_ES.UTF-8', 'ca_ES.ISO8859-1'), + ] + oldloc = locale.setlocale(locale.LC_ALL) + self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc) + # Wednesday, March: non-ASCII in all tested locales + # (including Turkish and German) + tt = (2026, 3, 4, 12, 34, 56, 2, 63, 0) + tested = False + for utf8_loc, legacy_loc in pairs: + try: + locale.setlocale(locale.LC_ALL, utf8_loc) + except locale.Error: + continue + expected = time.strftime('%A, %B', tt) + try: + locale.setlocale(locale.LC_ALL, legacy_loc) + except locale.Error: + continue + with self.subTest(locale=legacy_loc): + self.assertEqual(time.strftime('%A, %B', tt), expected) + tested = True + if not tested: + self.skipTest('no suitable locales') + def test_strftime_invalid_format(self): tt = time.gmtime(self.t) with SuppressCrashReport(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst new file mode 100644 index 000000000000000..cfdc812d7c6c249 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst @@ -0,0 +1,10 @@ +Fix corruption of locale-encoded text on FreeBSD, NetBSD, DragonFly BSD +and macOS +in non-UTF-8 locales, +e.g. in the results of :func:`time.strftime` and :func:`locale.nl_langinfo`, +in localized error messages, +and in decoding of the command line arguments, +fix wrong collation in :func:`locale.strcoll` and :func:`locale.strxfrm`, +and fix display and input of non-ASCII characters in :mod:`curses`. +``wchar_t`` values are not Unicode code points on these platforms, +so ``iconv()`` is now used instead of ``mbstowcs()`` and ``wcstombs()``. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index b2d745332317a36..73a9a8ad664d767 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -527,6 +527,11 @@ PyCurses_ConvertToWideCell(PyObject *obj, wchar_t *wch) (int)CCHARW_MAX, PyUnicode_GET_LENGTH(obj)); return -1; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wch, nch) < 0) { + return -1; + } +#endif /* A lone control character is allowed (like addch(ord('\n'))), but in a multi-character cell the base must be a printable spacing character and the rest zero-width combining characters. Check explicitly: otherwise @@ -622,6 +627,13 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, *wstr = PyUnicode_AsWideCharString(obj, NULL); if (*wstr == NULL) return 0; +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(*wstr, wcslen(*wstr)) < 0) { + PyMem_Free(*wstr); + *wstr = NULL; + return 0; + } +#endif return 2; #else assert (wstr == NULL); @@ -880,6 +892,9 @@ curses_cell_text(cursesmodule_state *state, const curses_cell_t *cell) PyErr_SetString(state->error, "getcchar() returned ERR"); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(wstr, wcslen(wstr)); +#endif return PyUnicode_FromWideChar(wstr, -1); #else char ch = (char)(*cell & A_CHARTEXT); @@ -1296,6 +1311,12 @@ complexstr_from_string(cursesmodule_state *state, PyObject *str, if (wbuf == NULL) { return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wbuf, n) < 0) { + PyMem_Free(wbuf); + return NULL; + } +#endif curses_cell_t *cells = n > 0 ? PyMem_New(curses_cell_t, n) : NULL; if (n > 0 && cells == NULL) { PyMem_Free(wbuf); @@ -1585,6 +1606,9 @@ complexstr_str(PyObject *self) } pos += wcslen(buf + pos); } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(buf, pos); +#endif PyObject *res = PyUnicode_FromWideChar(buf, pos); PyMem_Free(buf); return res; @@ -3275,6 +3299,13 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, rtn += 256; } #endif +#endif +#ifdef _Py_NON_UNICODE_WCHAR_T + { + wchar_t wch = (wchar_t)rtn; + _Py_LocaleWcharToUnicode_InPlace(&wch, 1); + return PyUnicode_FromOrdinal(wch); + } #endif return PyUnicode_FromOrdinal(rtn); } else { @@ -3325,8 +3356,16 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, } if (ct == KEY_CODE_YES) return PyLong_FromLong(rtn); +#ifdef _Py_NON_UNICODE_WCHAR_T + else { + wchar_t wch = (wchar_t)rtn; + _Py_LocaleWcharToUnicode_InPlace(&wch, 1); + return PyUnicode_FromOrdinal(wch); + } +#else else return PyUnicode_FromOrdinal(rtn); +#endif #else /* Without the wide library, read one key with wgetch(): a value above 255 is a function key (returned as an int); a byte is decoded with the @@ -3801,6 +3840,9 @@ PyCursesWindow_get_wstr(PyObject *op, PyObject *args) for (Py_ssize_t i = 0; i < len; i++) { wbuf[i] = (wchar_t)buf[i]; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(wbuf, len); +#endif PyObject *res = PyUnicode_FromWideChar(wbuf, len); PyMem_Free(wbuf); PyMem_Free(buf); @@ -3866,6 +3908,9 @@ PyCursesWindow_in_wstr(PyObject *op, PyObject *args) PyMem_Free(buf); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(buf, wcslen(buf)); +#endif PyObject *res = PyUnicode_FromWideChar(buf, -1); PyMem_Free(buf); return res; @@ -5854,6 +5899,9 @@ _curses_erasewchar_impl(PyObject *module) curses_set_error(module, "erasewchar", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(&ch, 1); +#endif return PyUnicode_FromWideChar(&ch, 1); #else /* Without the wide library, decode the single-byte erase character @@ -7076,6 +7124,9 @@ _curses_killwchar_impl(PyObject *module) curses_set_error(module, "killwchar", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + _Py_LocaleWcharToUnicode_InPlace(&ch, 1); +#endif return PyUnicode_FromWideChar(&ch, 1); #else /* Without the wide library, decode the single-byte kill character @@ -8018,6 +8069,19 @@ _curses_wunctrl(PyObject *module, PyObject *ch) curses_set_null_error(module, "wunctrl", NULL); return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + { + /* wunctrl() returns a pointer to a static buffer; make a copy + before converting in place. */ + wchar_t wbuf[8]; + size_t len = wcslen(res); + if (len < Py_ARRAY_LENGTH(wbuf)) { + wcscpy(wbuf, res); + _Py_LocaleWcharToUnicode_InPlace(wbuf, len); + return PyUnicode_FromWideChar(wbuf, len); + } + } +#endif return PyUnicode_FromWideChar(res, -1); #else /* Without the wide library, fall back to the single-byte unctrl() and @@ -8107,6 +8171,10 @@ _curses_ungetch(PyObject *module, PyObject *ch) wchar_t wch; if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0) + return NULL; +#endif return curses_check_err(module, unget_wch(wch), "unget_wch", "ungetch"); } #endif @@ -8138,6 +8206,10 @@ _curses_unget_wch(PyObject *module, PyObject *ch) if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; #ifdef HAVE_NCURSESW +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0) + return NULL; +#endif return curses_check_err(module, unget_wch(wch), "unget_wch", NULL); #else /* Without the wide library there is no unget_wch(): encode the character as @@ -8226,6 +8298,12 @@ _curses_slk_set_impl(PyObject *module, int labnum, PyObject *label, if (wstr == NULL) { return NULL; } +#ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_UnicodeToLocaleWchar_InPlace(wstr, wcslen(wstr)) < 0) { + PyMem_Free(wstr); + return NULL; + } +#endif rtn = slk_wset(labnum, wstr, justify); PyMem_Free(wstr); #else diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 20783dc00f90d3c..be089cddebd4af7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -406,6 +406,25 @@ static PyObject * _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) /*[clinic end generated code: output=82ddc6d62c76d618 input=693cd02bcbf38dd8]*/ { +#ifdef _Py_NON_UNICODE_WCHAR_T + /* wchar_t is not Unicode in non-UTF-8 locales and the collation + tables are keyed by the raw locale values, so wcscoll() cannot + be used with Unicode wchar_t. Encode to the locale encoding + and use strcoll(). */ + PyObject *result = NULL; + PyObject *b1 = PyUnicode_EncodeLocale(os1, NULL); + if (b1 == NULL) { + return NULL; + } + PyObject *b2 = PyUnicode_EncodeLocale(os2, NULL); + if (b2 != NULL) { + result = PyLong_FromLong(strcoll(PyBytes_AS_STRING(b1), + PyBytes_AS_STRING(b2))); + Py_DECREF(b2); + } + Py_DECREF(b1); + return result; +#else PyObject *result = NULL; wchar_t *ws1 = NULL, *ws2 = NULL; @@ -423,6 +442,7 @@ _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) if (ws1) PyMem_Free(ws1); if (ws2) PyMem_Free(ws2); return result; +#endif } #endif @@ -441,6 +461,44 @@ static PyObject * _locale_strxfrm_impl(PyObject *module, PyObject *str) /*[clinic end generated code: output=3081866ebffc01af input=1378bbe6a88b4780]*/ { +#ifdef _Py_NON_UNICODE_WCHAR_T + /* wchar_t is not Unicode in non-UTF-8 locales and the collation + tables are keyed by the raw locale values, so wcsxfrm() cannot + be used with Unicode wchar_t. Encode to the locale encoding + and use strxfrm(). The result is decoded as Latin-1: it maps + bytes to code points in the same order, so comparison of the + resulting strings compares the byte sequences. */ + PyObject *bytes = PyUnicode_EncodeLocale(str, NULL); + if (bytes == NULL) { + return NULL; + } + const char *s = PyBytes_AS_STRING(bytes); + PyObject *result = NULL; + char dummy[1]; + errno = 0; + size_t n = strxfrm(dummy, s, 1); + if (errno && errno != ERANGE) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + char *buf = PyMem_Malloc(n + 1); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + errno = 0; + n = strxfrm(buf, s, n + 1); + if (errno) { + PyErr_SetFromErrno(PyExc_OSError); + } + else { + result = PyUnicode_DecodeLatin1(buf, n, NULL); + } + PyMem_Free(buf); +error: + Py_DECREF(bytes); + return result; +#else Py_ssize_t n1; wchar_t *s = NULL, *buf = NULL; wchar_t dummy[1]; @@ -528,6 +586,7 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) PyMem_Free(buf); PyMem_Free(s); return result; +#endif /* _Py_NON_UNICODE_WCHAR_T */ } #endif diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 70d7e1b3713687a..38c2703a9278056 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -770,6 +770,15 @@ the C library strftime function.\n" # undef HAVE_WCSFTIME #endif +// On FreeBSD, NetBSD, DragonFly BSD and macOS, wchar_t values are not +// Unicode code points in non-UTF-8 locales, so the result of wcsftime() +// cannot be used directly. Use strftime() and decode its result with +// PyUnicode_DecodeLocaleAndSize() (which handles such platforms). +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__) +# undef HAVE_WCSFTIME +#endif + #ifdef HAVE_WCSFTIME #define time_char wchar_t #define format_time wcsftime diff --git a/Python/fileutils.c b/Python/fileutils.c index 5cbfd8e6ce4fa08..538612fd0e1b58c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -454,6 +454,363 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, } #endif /* !HAVE_MBRTOWC */ +#ifdef _Py_NON_UNICODE_WCHAR_T +/* wchar_t is not Unicode in non-UTF-8 locales (see pycore_fileutils.h), + so mbstowcs() and wcstombs() cannot be used to convert locale-encoded + text. Use iconv() instead. */ +# include +# include // MB_LEN_MAX +/* Native-endian UTF-32: a raw array of Unicode code points (unlike + "UCS-4", it also validates the converted values). */ +# ifdef WORDS_BIGENDIAN +# define _Py_UTF32_NATIVE "UTF-32BE" +# else +# define _Py_UTF32_NATIVE "UTF-32LE" +# endif +/* Fall back to mbstowcs()/wcstombs(). */ +# define _Py_ICONV_FALLBACK (-9) + +static iconv_t +locale_iconv_open(int decode) +{ + const char *codeset = nl_langinfo(CODESET); + if (!codeset || !*codeset + || strcmp(codeset, "UTF-8") == 0 + || strcmp(codeset, "US-ASCII") == 0 + || strcmp(codeset, "646") == 0) + { + /* mbstowcs() and wcstombs() are correct for these encodings */ + return (iconv_t)-1; + } + return decode ? iconv_open(_Py_UTF32_NATIVE, codeset) + : iconv_open(codeset, _Py_UTF32_NATIVE); +} + +static int +decode_locale_iconv(const char *arg, wchar_t **wstr, size_t *wlen, + const char **reason, _Py_error_handler errors) +{ + int surrogateescape; + if (get_surrogateescape(errors, &surrogateescape) < 0) { + return -3; + } + + size_t argsize = strlen(arg); + int ascii = 1; + for (size_t i = 0; i < argsize; i++) { + if ((unsigned char)arg[i] >= 0x80) { + ascii = 0; + break; + } + } + if (ascii) { + return _Py_ICONV_FALLBACK; + } + + iconv_t cd = locale_iconv_open(1); + if (cd == (iconv_t)-1) { + return _Py_ICONV_FALLBACK; + } + + if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + iconv_close(cd); + return -1; + } + /* the output cannot be longer than the number of input bytes */ + wchar_t *res = PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t)); + if (res == NULL) { + iconv_close(cd); + return -1; + } + + char *in = (char *)arg; + size_t inleft = argsize; + wchar_t *wout = res; + while (inleft > 0) { + char *out = (char *)wout; + size_t outleft = (res + argsize - wout) * sizeof(wchar_t); + /* Cast through void*: iconv() declares its second argument as + "char **" on most systems, but "const char **" on some. */ + size_t converted = iconv(cd, (void *)&in, &inleft, &out, &outleft); + wout = (wchar_t *)out; + if (converted == (size_t)-1) { + if (errno != EILSEQ && errno != EINVAL) { + /* unexpected error: fall back to mbstowcs() */ + PyMem_RawFree(res); + iconv_close(cd); + return _Py_ICONV_FALLBACK; + } + if (!surrogateescape) { + PyMem_RawFree(res); + iconv_close(cd); + if (wlen != NULL) { + *wlen = in - arg; + } + if (reason != NULL) { + *reason = "decoding error"; + } + return -2; + } + /* Escape the byte as UTF-8b, and start over in the initial + shift state. */ + *wout++ = 0xdc00 + (unsigned char)*in++; + inleft--; + iconv(cd, NULL, NULL, NULL, NULL); + } + } + iconv_close(cd); + *wout = L'\0'; + *wstr = res; + if (wlen != NULL) { + *wlen = wout - res; + } + return 0; +} + +static int +encode_locale_iconv(const wchar_t *text, char **str, + size_t *error_pos, const char **reason, + int raw_malloc, _Py_error_handler errors) +{ + int surrogateescape; + if (get_surrogateescape(errors, &surrogateescape) < 0) { + return -3; + } + + size_t len = wcslen(text); + int ascii = 1; + for (size_t i = 0; i < len; i++) { + if ((unsigned long)text[i] >= 0x80) { + ascii = 0; + break; + } + } + if (ascii) { + return _Py_ICONV_FALLBACK; + } + + iconv_t cd = locale_iconv_open(0); + if (cd == (iconv_t)-1) { + return _Py_ICONV_FALLBACK; + } + + char *result = NULL; + int res = -1; + if (len > (PY_SSIZE_T_MAX - MB_LEN_MAX - 1) / MB_LEN_MAX) { + goto done; + } + /* reserve room for the final shift sequence and the null byte */ + size_t size = len * MB_LEN_MAX + MB_LEN_MAX + 1; + result = raw_malloc ? PyMem_RawMalloc(size) : PyMem_Malloc(size); + if (result == NULL) { + goto done; + } + + const wchar_t *win = text; + const wchar_t *wend = text + len; + char *out = result; + size_t outleft = size - 1; + while (win < wend) { + wchar_t c = *win; + if (c >= 0xdc80 && c <= 0xdcff) { + if (!surrogateescape) { + goto encode_error; + } + /* UTF-8b surrogate: write the raw byte, and start over in + the initial shift state */ + *out++ = (char)(c - 0xdc00); + outleft--; + win++; + iconv(cd, NULL, NULL, NULL, NULL); + continue; + } + /* Note: iconv() can replace a character which cannot be encoded + with an approximation (e.g. "'e" for "é" or "?"), depending + on the implementation. */ + char *in = (char *)win; + size_t inleft = (wend - win) * sizeof(wchar_t); + /* Cast through void*: iconv() declares its second argument as + "char **" on most systems, but "const char **" on some. */ + size_t converted = iconv(cd, (void *)&in, &inleft, &out, &outleft); + win = (const wchar_t *)in; + if (converted == (size_t)-1) { + if (errno != EILSEQ && errno != EINVAL) { + /* unexpected error: fall back to wcstombs() */ + goto fallback; + } + /* iconv() stopped at the first character which cannot be + encoded; an escaped surrogate is handled at the start of + the next iteration */ + if (!(*win >= 0xdc80 && *win <= 0xdcff)) { + goto encode_error; + } + } + } + /* write the final shift sequence for stateful encodings */ + if (iconv(cd, NULL, NULL, &out, &outleft) == (size_t)-1) { + goto fallback; + } + *out = '\0'; + *str = result; + result = NULL; + res = 0; + goto done; + +fallback: + res = _Py_ICONV_FALLBACK; + goto done; + +encode_error: + res = -2; + if (error_pos != NULL) { + *error_pos = win - text; + } + if (reason != NULL) { + *reason = "encoding error"; + } + +done: + if (result != NULL) { + if (raw_malloc) { + PyMem_RawFree(result); + } + else { + PyMem_Free(result); + } + } + iconv_close(cd); + return res; +} + +/* Convert a wide character string in place from Unicode code points to + the native wchar_t form, for passing to C library functions which + expect the latter (such as the curses library). + Return 0 on success. Return -1 with an exception set if a character + cannot be represented in the locale encoding. */ +int +_Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size) +{ + iconv_t cd = (iconv_t)-1; + int res = 0; + for (Py_ssize_t i = 0; i < size; i++) { + wchar_t wc = str[i]; + if ((unsigned long)wc < 0x80 + || ((unsigned long)wc >= 0xdc80 && (unsigned long)wc <= 0xdcff)) + { + continue; + } + if (cd == (iconv_t)-1) { + cd = locale_iconv_open(0); + if (cd == (iconv_t)-1) { + /* wchar_t is Unicode for this locale encoding, or no + conversion is possible */ + return 0; + } + } + else { + /* reset cd to the initial shift state */ + iconv(cd, NULL, NULL, NULL, NULL); + } + Py_UCS4 ch = (Py_UCS4)wc; + char buf[MB_LEN_MAX]; + char *in = (char *)&ch; + size_t inleft = sizeof(ch); + char *out = buf; + size_t outleft = sizeof(buf); + mbstate_t mbs; + wchar_t converted; + size_t blen; + /* A non-zero result means that the character was replaced + with a substitute. */ + if (iconv(cd, (void *)&in, &inleft, &out, &outleft) != 0 + || inleft != 0) + { + goto encode_error; + } + /* locale bytes -> native form: exactly one wchar_t */ + blen = sizeof(buf) - outleft; + memset(&mbs, 0, sizeof(mbs)); + if (mbrtowc(&converted, buf, blen, &mbs) != blen) { + goto encode_error; + } + str[i] = converted; + continue; + +encode_error: + { + PyObject *obj = PyUnicode_FromOrdinal((int)wc); + if (obj != NULL) { + PyObject *exc = PyObject_CallFunction( + PyExc_UnicodeEncodeError, "sOnns", + nl_langinfo(CODESET), obj, + (Py_ssize_t)0, (Py_ssize_t)1, + "character maps to "); + if (exc != NULL) { + PyCodec_StrictErrors(exc); + Py_DECREF(exc); + } + Py_DECREF(obj); + } + } + res = -1; + break; + } + if (cd != (iconv_t)-1) { + iconv_close(cd); + } + return res; +} + +/* The reverse conversion: from the native wchar_t form to Unicode + code points. Values which cannot be converted are left unchanged. */ +void +_Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size) +{ + iconv_t cd = (iconv_t)-1; + for (Py_ssize_t i = 0; i < size; i++) { + wchar_t wc = str[i]; + if ((unsigned long)wc < 0x80 + || ((unsigned long)wc >= 0xdc80 && (unsigned long)wc <= 0xdcff)) + { + continue; + } + /* native form -> locale bytes */ + char buf[MB_LEN_MAX]; + mbstate_t mbs; + memset(&mbs, 0, sizeof(mbs)); + size_t blen = wcrtomb(buf, wc, &mbs); + if (blen == (size_t)-1) { + continue; + } + /* locale bytes -> Unicode */ + if (cd == (iconv_t)-1) { + cd = locale_iconv_open(1); + if (cd == (iconv_t)-1) { + return; + } + } + else { + /* reset cd to the initial shift state */ + iconv(cd, NULL, NULL, NULL, NULL); + } + char *in = buf; + size_t inleft = blen; + Py_UCS4 ch; + char *out = (char *)&ch; + size_t outleft = sizeof(ch); + if (iconv(cd, (void *)&in, &inleft, &out, &outleft) == (size_t)-1 + || inleft != 0 || outleft != 0) + { + continue; + } + str[i] = (wchar_t)ch; + } + if (cd != (iconv_t)-1) { + iconv_close(cd); + } +} +#endif /* _Py_NON_UNICODE_WCHAR_T */ + static int decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) @@ -467,6 +824,13 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, mbstate_t mbs; #endif +#ifdef _Py_NON_UNICODE_WCHAR_T + int iconv_res = decode_locale_iconv(arg, wstr, wlen, reason, errors); + if (iconv_res != _Py_ICONV_FALLBACK) { + return iconv_res; + } +#endif + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3; @@ -683,6 +1047,14 @@ encode_current_locale(const wchar_t *text, char **str, size_t i, size, converted; wchar_t c, buf[2]; +#ifdef _Py_NON_UNICODE_WCHAR_T + int iconv_res = encode_locale_iconv(text, str, error_pos, reason, + raw_malloc, errors); + if (iconv_res != _Py_ICONV_FALLBACK) { + return iconv_res; + } +#endif + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3; From 44f96dfb02b3ec0e34ee526d4d8a2a1a6d490734 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jul 2026 15:44:32 +0300 Subject: [PATCH 2/4] Do not fail if iconv() cannot decode what mbstowcs() can iconv() can be stricter than mbstowcs(), and a broken locale should not make the conversion fail, so fall back to mbstowcs() instead of raising UnicodeDecodeError. Co-Authored-By: Claude Fable 5 --- Python/fileutils.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 538612fd0e1b58c..689916c34142f12 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -541,15 +541,12 @@ decode_locale_iconv(const char *arg, wchar_t **wstr, size_t *wlen, return _Py_ICONV_FALLBACK; } if (!surrogateescape) { + /* Leave it to mbstowcs() to decide whether the input is + valid: it can be more permissive than iconv(), and a + broken locale should not make the conversion fail. */ PyMem_RawFree(res); iconv_close(cd); - if (wlen != NULL) { - *wlen = in - arg; - } - if (reason != NULL) { - *reason = "decoding error"; - } - return -2; + return _Py_ICONV_FALLBACK; } /* Escape the byte as UTF-8b, and start over in the initial shift state. */ From 94ee8e2e531220d80bde3c7b7abccc57ea85605d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jul 2026 15:44:32 +0300 Subject: [PATCH 3/4] Fix the decimal point and the thousands separator in the decimal module dotsep_as_utf8() used mbstowcs(), which does not produce Unicode code points in non-UTF-8 locales on the affected platforms. Co-Authored-By: Claude Fable 5 --- Modules/_decimal/_decimal.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 6fbce0ed85e695d..4f88ad7f1b54747 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3657,20 +3657,20 @@ dotsep_as_utf8(const char *s) { PyObject *utf8; PyObject *tmp; - wchar_t buf[2]; - size_t n; - n = mbstowcs(buf, s, 2); - if (n != 1) { /* Issue #7442 */ + /* Do not use mbstowcs(): wchar_t values are not Unicode code points + in non-UTF-8 locales on some platforms (e.g. the BSDs and macOS). */ + tmp = PyUnicode_DecodeLocale(s, NULL); + if (tmp == NULL) { + return NULL; + } + if (PyUnicode_GET_LENGTH(tmp) != 1) { /* Issue #7442 */ + Py_DECREF(tmp); PyErr_SetString(PyExc_ValueError, "invalid decimal point or unsupported " "combination of LC_CTYPE and LC_NUMERIC"); return NULL; } - tmp = PyUnicode_FromWideChar(buf, n); - if (tmp == NULL) { - return NULL; - } utf8 = PyUnicode_AsUTF8String(tmp); Py_DECREF(tmp); return utf8; From 423e68c7483377a9ef1a5e0efbdf7e8bbf1775ce Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jul 2026 18:30:29 +0300 Subject: [PATCH 4/4] Avoid the overhead of the conversion in UTF-8 and ASCII locales Add _Py_LocaleNeedsWcharConversion(), which memoizes the check of the locale encoding, and test it before scanning the string. Also keep using wcscoll() and wcsxfrm() when no conversion is needed, so that locale.strxfrm() returns the same keys as before. Co-Authored-By: Claude Fable 5 --- Include/internal/pycore_fileutils.h | 1 + Include/internal/pycore_runtime_structs.h | 6 +++ Modules/_localemodule.c | 14 +++++-- Python/fileutils.c | 48 ++++++++++++++++++++--- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 31807cfb98ca05a..36faef0620b52d6 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -243,6 +243,7 @@ extern void _Py_ResetForceASCII(void); // Convert between Unicode code points and the native wchar_t form for // C library functions which use the latter (such as the curses library). // Export for '_curses'. +PyAPI_FUNC(int) _Py_LocaleNeedsWcharConversion(void); PyAPI_FUNC(int) _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size); PyAPI_FUNC(void) _Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size); #endif diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 145e66de9984ca7..f1dccd82c5b5b7e 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -52,6 +52,12 @@ struct pyhash_runtime_state { struct _fileutils_state { int force_ascii; + /* Memoized result of _Py_LocaleNeedsWcharConversion() with the locale + encoding it was computed for (see Python/fileutils.c). */ + struct _Py_wchar_conversion_state { + char codeset[32]; + int needed; + } wchar; }; #include "pycore_interpframe_structs.h" // _PyInterpreterFrame diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index be089cddebd4af7..5043069dad047f2 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -407,6 +407,7 @@ _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) /*[clinic end generated code: output=82ddc6d62c76d618 input=693cd02bcbf38dd8]*/ { #ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_LocaleNeedsWcharConversion()) { /* wchar_t is not Unicode in non-UTF-8 locales and the collation tables are keyed by the raw locale values, so wcscoll() cannot be used with Unicode wchar_t. Encode to the locale encoding @@ -424,7 +425,9 @@ _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) } Py_DECREF(b1); return result; -#else + } +#endif + { PyObject *result = NULL; wchar_t *ws1 = NULL, *ws2 = NULL; @@ -442,7 +445,7 @@ _locale_strcoll_impl(PyObject *module, PyObject *os1, PyObject *os2) if (ws1) PyMem_Free(ws1); if (ws2) PyMem_Free(ws2); return result; -#endif + } } #endif @@ -462,6 +465,7 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) /*[clinic end generated code: output=3081866ebffc01af input=1378bbe6a88b4780]*/ { #ifdef _Py_NON_UNICODE_WCHAR_T + if (_Py_LocaleNeedsWcharConversion()) { /* wchar_t is not Unicode in non-UTF-8 locales and the collation tables are keyed by the raw locale values, so wcsxfrm() cannot be used with Unicode wchar_t. Encode to the locale encoding @@ -498,7 +502,9 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) error: Py_DECREF(bytes); return result; -#else + } +#endif + { Py_ssize_t n1; wchar_t *s = NULL, *buf = NULL; wchar_t dummy[1]; @@ -586,7 +592,7 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) PyMem_Free(buf); PyMem_Free(s); return result; -#endif /* _Py_NON_UNICODE_WCHAR_T */ + } } #endif diff --git a/Python/fileutils.c b/Python/fileutils.c index 689916c34142f12..c858e6e2604c31e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -470,15 +470,37 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, /* Fall back to mbstowcs()/wcstombs(). */ # define _Py_ICONV_FALLBACK (-9) +/* Return non-zero if wchar_t is not Unicode in the current locale, i.e. if + the locale encoding is neither UTF-8 nor ASCII. The result is memoized: + this is called before every conversion, and in the common case (a UTF-8 + locale) it is the only overhead. */ +int +_Py_LocaleNeedsWcharConversion(void) +{ + const char *codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + return 0; + } + struct _Py_wchar_conversion_state *state = &_PyRuntime.fileutils.wchar; + if (strcmp(codeset, state->codeset) == 0) { + return state->needed; + } + int needed = (*codeset != '\0' + && strcmp(codeset, "UTF-8") != 0 + && strcmp(codeset, "US-ASCII") != 0 + && strcmp(codeset, "646") != 0); + if (strlen(codeset) < sizeof(state->codeset)) { + memcpy(state->codeset, codeset, strlen(codeset) + 1); + state->needed = needed; + } + return needed; +} + static iconv_t locale_iconv_open(int decode) { const char *codeset = nl_langinfo(CODESET); - if (!codeset || !*codeset - || strcmp(codeset, "UTF-8") == 0 - || strcmp(codeset, "US-ASCII") == 0 - || strcmp(codeset, "646") == 0) - { + if (!_Py_LocaleNeedsWcharConversion()) { /* mbstowcs() and wcstombs() are correct for these encodings */ return (iconv_t)-1; } @@ -490,6 +512,10 @@ static int decode_locale_iconv(const char *arg, wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) { + if (!_Py_LocaleNeedsWcharConversion()) { + return _Py_ICONV_FALLBACK; + } + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3; @@ -569,6 +595,10 @@ encode_locale_iconv(const wchar_t *text, char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { + if (!_Py_LocaleNeedsWcharConversion()) { + return _Py_ICONV_FALLBACK; + } + int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { return -3; @@ -687,6 +717,10 @@ encode_locale_iconv(const wchar_t *text, char **str, int _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size) { + if (!_Py_LocaleNeedsWcharConversion()) { + return 0; + } + iconv_t cd = (iconv_t)-1; int res = 0; for (Py_ssize_t i = 0; i < size; i++) { @@ -763,6 +797,10 @@ _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size) void _Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size) { + if (!_Py_LocaleNeedsWcharConversion()) { + return; + } + iconv_t cd = (iconv_t)-1; for (Py_ssize_t i = 0; i < size; i++) { wchar_t wc = str[i];