From e03cae6ad4b929b20174f3ca69cfb6b1f98c63c3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 2 Jul 2026 20:33:11 +0300 Subject: [PATCH 1/3] gh-152905: Decode LC_TIME items in nl_langinfo() from glibc wide data On glibc, locale.nl_langinfo() now decodes the LC_TIME text items from the wide (_NL_W*) locale data, independently of the LC_CTYPE encoding. Co-Authored-By: Claude Opus 4.8 (1M context) --- Doc/library/locale.rst | 4 + Lib/test/test__locale.py | 50 +++++++++++ ...-07-01-23-17-27.gh-issue-152905.wLnGqR.rst | 3 + Modules/_localemodule.c | 90 ++++++++++++++----- 4 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index e02cbe7d669f8b..8909d9d7b187e0 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -342,6 +342,10 @@ The :mod:`!locale` module defines the following exception and functions: .. versionchanged:: 3.14 The function now temporarily sets the ``LC_CTYPE`` locale in some cases. + .. versionchanged:: next + On glibc, the ``LC_TIME`` items are now decoded + independently of the ``LC_CTYPE`` encoding. + .. function:: getdefaultlocale([envvars]) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 29a29ca0c65097..ec91eaf396dacf 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -271,6 +271,56 @@ def test_era_nl_langinfo(self): if not tested: self.skipTest('no suitable locales') + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_nl_langinfo_encoding_independent(self): + # gh-152905: The LC_TIME text items are decoded independently of the + # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so + # the same locale in different encodings yields identical strings. + self.addCleanup(setlocale, LC_TIME, setlocale(LC_TIME)) + months = [getattr(locale, f'MON_{i}') for i in range(1, 13)] + items = months.copy() + items += [getattr(locale, f'ABMON_{i}') for i in range(1, 13)] + items += [getattr(locale, f'DAY_{i}') for i in range(1, 8)] + items += [getattr(locale, f'ABDAY_{i}') for i in range(1, 8)] + items += [locale.AM_STR, locale.PM_STR, + locale.D_T_FMT, locale.D_FMT, locale.T_FMT] + if hasattr(locale, 'T_FMT_AMPM'): + items.append(locale.T_FMT_AMPM) + if hasattr(locale, 'ALT_DIGITS'): + items.append(locale.ALT_DIGITS) + # The same language in a Unicode and a legacy encoding. + variants = [ + ('ja_JP.UTF-8', 'ja_JP.EUC-JP'), + ('fr_FR.UTF-8', 'fr_FR.ISO8859-1'), + ('el_GR.UTF-8', 'el_GR.ISO8859-7'), + ] + tested = False + for locs in variants: + values = [] + avail = [] + for loc in locs: + try: + setlocale(LC_TIME, loc) + except Error: + continue + avail.append(loc) + values.append([nl_langinfo(item) for item in items]) + if len(values) < 2: + continue + with self.subTest(locales=avail): + for value in values[0]: + self.assertIsInstance(value, str) + # Month names within a locale must be distinct, which guards + # against a broken narrow-to-wide constant mapping. + self.assertEqual(len(set(values[0][:12])), 12, values[0][:12]) + # The result must not depend on the locale encoding. + for other in values[1:]: + self.assertEqual(values[0], other) + tested = True + if not tested: + self.skipTest('no suitable locale pairs') + def test_float_parsing(self): # Bug #1391872: Test whether float parsing is okay on European # locales. diff --git a/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst b/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst new file mode 100644 index 00000000000000..99f20e65ad8e08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst @@ -0,0 +1,3 @@ +On glibc, :func:`locale.nl_langinfo` now decodes the ``LC_TIME`` items (such +as the month and day names) using the wide locale data, so the result no +longer depends on the ``LC_CTYPE`` encoding. diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 8f7d662b00b21b..1408ba133b32b5 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -561,11 +561,23 @@ _locale__getdefaultlocale_impl(PyObject *module) #endif #ifdef HAVE_LANGINFO_H -#define LANGINFO(X, Y) {#X, X, Y} +/* On glibc, LC_TIME text items also have a wide (_NL_W*) form, named + _NL_W, that nl_langinfo() returns as wchar_t*. LANGINFO_NW + marks the items that have no wide counterpart. */ +#ifdef __GLIBC__ +# define LANGINFO(X, Y) {#X, X, Y, _NL_W ## X} +# define LANGINFO_NW(X, Y) {#X, X, Y, 0} +#else +# define LANGINFO(X, Y) {#X, X, Y} +# define LANGINFO_NW(X, Y) {#X, X, Y} +#endif static struct langinfo_constant{ const char *name; int value; int category; +#ifdef __GLIBC__ + nl_item wide; /* wide _NL_W* counterpart, or 0 if none */ +#endif } langinfo_constants[] = { /* These constants should exist on any langinfo implementation */ @@ -613,8 +625,8 @@ static struct langinfo_constant{ #ifdef RADIXCHAR /* The following are not available with glibc 2.0 */ - LANGINFO(RADIXCHAR, LC_NUMERIC), - LANGINFO(THOUSEP, LC_NUMERIC), + LANGINFO_NW(RADIXCHAR, LC_NUMERIC), + LANGINFO_NW(THOUSEP, LC_NUMERIC), /* YESSTR and NOSTR are deprecated in glibc, since they are a special case of message translation, which should be rather done using gettext. So we don't expose it to Python in the @@ -622,7 +634,7 @@ static struct langinfo_constant{ LANGINFO(YESSTR, LC_MESSAGES), LANGINFO(NOSTR, LC_MESSAGES), */ - LANGINFO(CRNCYSTR, LC_MONETARY), + LANGINFO_NW(CRNCYSTR, LC_MONETARY), #endif LANGINFO(D_T_FMT, LC_TIME), @@ -636,13 +648,13 @@ static struct langinfo_constant{ a few of the others. Solution: ifdef-test them all. */ #ifdef CODESET - LANGINFO(CODESET, LC_CTYPE), + LANGINFO_NW(CODESET, LC_CTYPE), #endif #ifdef T_FMT_AMPM LANGINFO(T_FMT_AMPM, LC_TIME), #endif #ifdef ERA - LANGINFO(ERA, LC_TIME), + LANGINFO_NW(ERA, LC_TIME), #endif #ifdef ERA_D_FMT LANGINFO(ERA_D_FMT, LC_TIME), @@ -657,10 +669,10 @@ static struct langinfo_constant{ LANGINFO(ALT_DIGITS, LC_TIME), #endif #ifdef YESEXPR - LANGINFO(YESEXPR, LC_MESSAGES), + LANGINFO_NW(YESEXPR, LC_MESSAGES), #endif #ifdef NOEXPR - LANGINFO(NOEXPR, LC_MESSAGES), + LANGINFO_NW(NOEXPR, LC_MESSAGES), #endif #ifdef _DATE_FMT /* This is not available in all glibc versions that have CODESET. */ @@ -709,7 +721,6 @@ restore_locale(char *oldloc) } #ifdef __GLIBC__ -#if defined(ALT_DIGITS) || defined(ERA) static PyObject * decode_strings(const char *result, size_t max_count) { @@ -737,7 +748,35 @@ decode_strings(const char *result, size_t max_count) return pyresult; } #endif -#endif + +#ifdef __GLIBC__ +static PyObject * +decode_wide_strings(const wchar_t *result, size_t max_count) +{ + /* Wide-character counterpart of decode_strings(): convert a sequence of + * NUL-separated wchar_t strings to a str with semicolon-separated items. */ + size_t i = 0; + size_t count = 0; + for (; count < max_count && result[i]; count++) { + i += wcslen(result + i) + 1; + } + wchar_t *buf = PyMem_New(wchar_t, i); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + memcpy(buf, result, i * sizeof(wchar_t)); + /* Replace all NULs with semicolons. */ + i = 0; + while (--count) { + i += wcslen(buf + i); + buf[i++] = L';'; + } + PyObject *pyresult = PyUnicode_FromWideChar(buf, -1); + PyMem_Free(buf); + return pyresult; +} +#endif /* __GLIBC__ */ /*[clinic input] _locale.nl_langinfo @@ -758,6 +797,22 @@ _locale_nl_langinfo_impl(PyObject *module, int item) crash PyUnicode_FromString. */ for (i = 0; langinfo_constants[i].name; i++) { if (langinfo_constants[i].value == item) { +#ifdef __GLIBC__ + /* Prefer the wide variant: it is decoded independently of the + LC_CTYPE encoding. */ + nl_item wide = langinfo_constants[i].wide; + if (wide) { + const wchar_t *wresult = (const wchar_t *)nl_langinfo(wide); + if (wresult == NULL) { + wresult = L""; + } + /* ALT_DIGITS is a sequence of NUL-separated strings. */ + if (item == ALT_DIGITS && *wresult) { + return decode_wide_strings(wresult, 100); + } + return PyUnicode_FromWideChar(wresult, -1); + } +#endif /* Check NULL as a workaround for GNU libc's returning NULL instead of an empty string for nl_langinfo(ERA). */ const char *result = nl_langinfo(item); @@ -766,13 +821,8 @@ _locale_nl_langinfo_impl(PyObject *module, int item) if (langinfo_constants[i].category != LC_CTYPE && *result && ( #ifdef __GLIBC__ - // gh-133740: Always change the locale for ALT_DIGITS and ERA -# ifdef ALT_DIGITS - item == ALT_DIGITS || -# endif -# ifdef ERA + // gh-133740: Always change the locale for ERA item == ERA || -# endif #endif !is_all_ascii(result)) && change_locale(langinfo_constants[i].category, &oldloc) < 0) @@ -784,18 +834,10 @@ _locale_nl_langinfo_impl(PyObject *module, int item) /* According to the POSIX specification the result must be * a sequence of semicolon-separated strings. * But in Glibc they are NUL-separated. */ -#ifdef ALT_DIGITS - if (item == ALT_DIGITS && *result) { - pyresult = decode_strings(result, 100); - } - else -#endif -#ifdef ERA if (item == ERA && *result) { pyresult = decode_strings(result, SIZE_MAX); } else -#endif #endif { pyresult = PyUnicode_DecodeLocale(result, NULL); From 331153211591438e02d044b50a50fc7ac1cb9d90 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 3 Jul 2026 08:40:21 +0300 Subject: [PATCH 2/3] gh-152905: Skip the new nl_langinfo test on non-glibc The encoding-independence guaranteed by the wide (_NL_W*) decode is glibc-specific, so gate test_nl_langinfo_encoding_independent on glibc (which also covers the previously skipped musl case). Co-Authored-By: Claude Opus 4.8 (1M context) --- Lib/test/test__locale.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index ec91eaf396dacf..ff4b1cb84377f7 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -7,7 +7,7 @@ import locale import sys import unittest -from platform import uname +from platform import uname, libc_ver from test import support @@ -272,7 +272,8 @@ def test_era_nl_langinfo(self): self.skipTest('no suitable locales') @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") - @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipUnless(libc_ver()[0] == 'glibc', + "wide nl_langinfo variants are glibc-specific") def test_nl_langinfo_encoding_independent(self): # gh-152905: The LC_TIME text items are decoded independently of the # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so From 85d729d12cbd9a3612d61d1d4152e1fcc3dad659 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 3 Jul 2026 20:50:15 +0300 Subject: [PATCH 3/3] gh-152905: Address review of nl_langinfo wide LC_TIME decoding * Rewrite test_nl_langinfo_encoding_independent to compare each item individually (clearer failures), listing only the legacy locales and deriving the UTF-8 variant; broaden coverage to 20 locales across 17 legacy encodings. Also test ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT and _DATE_FMT; note that ERA has no wide variant and is not tested. * Drop the now-unused max_count parameter of decode_strings(). * Mention in the docs that ERA is not affected. Co-Authored-By: Claude Opus 4.8 --- Doc/library/locale.rst | 2 +- Lib/test/test__locale.py | 80 +++++++++++++++++++++++----------------- Modules/_localemodule.c | 6 +-- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 8909d9d7b187e0..933ad039b6505f 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -343,7 +343,7 @@ The :mod:`!locale` module defines the following exception and functions: The function now temporarily sets the ``LC_CTYPE`` locale in some cases. .. versionchanged:: next - On glibc, the ``LC_TIME`` items are now decoded + On glibc, the ``LC_TIME`` items (except ``ERA``) are now decoded independently of the ``LC_CTYPE`` encoding. diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index ff4b1cb84377f7..2569161d3507a5 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -1,3 +1,4 @@ +import _locale from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) try: from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) @@ -278,47 +279,60 @@ def test_nl_langinfo_encoding_independent(self): # gh-152905: The LC_TIME text items are decoded independently of the # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so # the same locale in different encodings yields identical strings. + # ERA has no wide variant and is not tested. self.addCleanup(setlocale, LC_TIME, setlocale(LC_TIME)) - months = [getattr(locale, f'MON_{i}') for i in range(1, 13)] - items = months.copy() - items += [getattr(locale, f'ABMON_{i}') for i in range(1, 13)] - items += [getattr(locale, f'DAY_{i}') for i in range(1, 8)] - items += [getattr(locale, f'ABDAY_{i}') for i in range(1, 8)] - items += [locale.AM_STR, locale.PM_STR, - locale.D_T_FMT, locale.D_FMT, locale.T_FMT] - if hasattr(locale, 'T_FMT_AMPM'): - items.append(locale.T_FMT_AMPM) - if hasattr(locale, 'ALT_DIGITS'): - items.append(locale.ALT_DIGITS) - # The same language in a Unicode and a legacy encoding. - variants = [ - ('ja_JP.UTF-8', 'ja_JP.EUC-JP'), - ('fr_FR.UTF-8', 'fr_FR.ISO8859-1'), - ('el_GR.UTF-8', 'el_GR.ISO8859-7'), + + names = [f'MON_{i}' for i in range(1, 13)] + names += [f'ABMON_{i}' for i in range(1, 13)] + names += [f'DAY_{i}' for i in range(1, 8)] + names += [f'ABDAY_{i}' for i in range(1, 8)] + names += ['AM_STR', 'PM_STR', 'D_T_FMT', 'D_FMT', 'T_FMT'] + names += [name for name in ('T_FMT_AMPM', 'ERA_D_FMT', 'ERA_D_T_FMT', + 'ERA_T_FMT', 'ALT_DIGITS', '_DATE_FMT') + if hasattr(_locale, name)] + items = [(name, getattr(_locale, name)) for name in names] + + # Legacy (non-UTF-8) locales, compared against their UTF-8 variant. + legacy_locales = [ + 'en_US.ISO8859-1', + 'es_ES.ISO8859-1', + 'fr_FR.ISO8859-1', + 'de_DE.ISO8859-1', + 'pl_PL.ISO8859-2', + 'mt_MT.ISO8859-3', + 'ar_SA.ISO8859-6', + 'el_GR.ISO8859-7', + 'he_IL.ISO8859-8', + 'tr_TR.ISO8859-9', + 'lt_LT.ISO8859-13', + 'cy_GB.ISO8859-14', + 'et_EE.ISO8859-15', + 'uk_UA.KOI8-U', + 'bg_BG.CP1251', + 'ja_JP.EUC-JP', + 'ko_KR.EUC-KR', + 'zh_CN.GB2312', + 'zh_TW.BIG5', + 'th_TH.TIS-620', ] tested = False - for locs in variants: + for loc in legacy_locales: + locs = (loc.partition('.')[0] + '.UTF-8', loc) values = [] - avail = [] - for loc in locs: + for l in locs: try: - setlocale(LC_TIME, loc) + setlocale(LC_TIME, l) except Error: - continue - avail.append(loc) - values.append([nl_langinfo(item) for item in items]) + break + values.append({name: nl_langinfo(item) for name, item in items}) if len(values) < 2: continue - with self.subTest(locales=avail): - for value in values[0]: - self.assertIsInstance(value, str) - # Month names within a locale must be distinct, which guards - # against a broken narrow-to-wide constant mapping. - self.assertEqual(len(set(values[0][:12])), 12, values[0][:12]) - # The result must not depend on the locale encoding. - for other in values[1:]: - self.assertEqual(values[0], other) - tested = True + tested = True + + # The result must not depend on the locale encoding. + for name, item in items: + with self.subTest(locales=locs, name=name): + self.assertEqual(values[0][name], values[1][name]) if not tested: self.skipTest('no suitable locale pairs') diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 1408ba133b32b5..0f71358a180dd2 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -722,13 +722,13 @@ restore_locale(char *oldloc) #ifdef __GLIBC__ static PyObject * -decode_strings(const char *result, size_t max_count) +decode_strings(const char *result) { /* Convert a sequence of NUL-separated C strings to a Python string * containing semicolon separated items. */ size_t i = 0; size_t count = 0; - for (; count < max_count && result[i]; count++) { + for (; result[i]; count++) { i += strlen(result + i) + 1; } char *buf = PyMem_Malloc(i); @@ -835,7 +835,7 @@ _locale_nl_langinfo_impl(PyObject *module, int item) * a sequence of semicolon-separated strings. * But in Glibc they are NUL-separated. */ if (item == ERA && *result) { - pyresult = decode_strings(result, SIZE_MAX); + pyresult = decode_strings(result); } else #endif