diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index e02cbe7d669f8b..933ad039b6505f 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -342,6 +342,10 @@ The :mod:`!locale` module defines the following exception and functions: .. versionchanged:: 3.14 The function now temporarily sets the ``LC_CTYPE`` locale in some cases. + .. versionchanged:: next + On glibc, the ``LC_TIME`` items (except ``ERA``) are now decoded + independently of the ``LC_CTYPE`` encoding. + .. function:: getdefaultlocale([envvars]) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 29a29ca0c65097..2569161d3507a5 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -1,3 +1,4 @@ +import _locale from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) try: from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) @@ -7,7 +8,7 @@ import locale import sys import unittest -from platform import uname +from platform import uname, libc_ver from test import support @@ -271,6 +272,70 @@ def test_era_nl_langinfo(self): if not tested: self.skipTest('no suitable locales') + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(libc_ver()[0] == 'glibc', + "wide nl_langinfo variants are glibc-specific") + def test_nl_langinfo_encoding_independent(self): + # gh-152905: The LC_TIME text items are decoded independently of the + # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so + # the same locale in different encodings yields identical strings. + # ERA has no wide variant and is not tested. + self.addCleanup(setlocale, LC_TIME, setlocale(LC_TIME)) + + names = [f'MON_{i}' for i in range(1, 13)] + names += [f'ABMON_{i}' for i in range(1, 13)] + names += [f'DAY_{i}' for i in range(1, 8)] + names += [f'ABDAY_{i}' for i in range(1, 8)] + names += ['AM_STR', 'PM_STR', 'D_T_FMT', 'D_FMT', 'T_FMT'] + names += [name for name in ('T_FMT_AMPM', 'ERA_D_FMT', 'ERA_D_T_FMT', + 'ERA_T_FMT', 'ALT_DIGITS', '_DATE_FMT') + if hasattr(_locale, name)] + items = [(name, getattr(_locale, name)) for name in names] + + # Legacy (non-UTF-8) locales, compared against their UTF-8 variant. + legacy_locales = [ + 'en_US.ISO8859-1', + 'es_ES.ISO8859-1', + 'fr_FR.ISO8859-1', + 'de_DE.ISO8859-1', + 'pl_PL.ISO8859-2', + 'mt_MT.ISO8859-3', + 'ar_SA.ISO8859-6', + 'el_GR.ISO8859-7', + 'he_IL.ISO8859-8', + 'tr_TR.ISO8859-9', + 'lt_LT.ISO8859-13', + 'cy_GB.ISO8859-14', + 'et_EE.ISO8859-15', + 'uk_UA.KOI8-U', + 'bg_BG.CP1251', + 'ja_JP.EUC-JP', + 'ko_KR.EUC-KR', + 'zh_CN.GB2312', + 'zh_TW.BIG5', + 'th_TH.TIS-620', + ] + tested = False + for loc in legacy_locales: + locs = (loc.partition('.')[0] + '.UTF-8', loc) + values = [] + for l in locs: + try: + setlocale(LC_TIME, l) + except Error: + break + values.append({name: nl_langinfo(item) for name, item in items}) + if len(values) < 2: + continue + tested = True + + # The result must not depend on the locale encoding. + for name, item in items: + with self.subTest(locales=locs, name=name): + self.assertEqual(values[0][name], values[1][name]) + if not tested: + self.skipTest('no suitable locale pairs') + def test_float_parsing(self): # Bug #1391872: Test whether float parsing is okay on European # locales. diff --git a/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst b/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst new file mode 100644 index 00000000000000..99f20e65ad8e08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-23-17-27.gh-issue-152905.wLnGqR.rst @@ -0,0 +1,3 @@ +On glibc, :func:`locale.nl_langinfo` now decodes the ``LC_TIME`` items (such +as the month and day names) using the wide locale data, so the result no +longer depends on the ``LC_CTYPE`` encoding. diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 8f7d662b00b21b..0f71358a180dd2 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -561,11 +561,23 @@ _locale__getdefaultlocale_impl(PyObject *module) #endif #ifdef HAVE_LANGINFO_H -#define LANGINFO(X, Y) {#X, X, Y} +/* On glibc, LC_TIME text items also have a wide (_NL_W*) form, named + _NL_W, that nl_langinfo() returns as wchar_t*. LANGINFO_NW + marks the items that have no wide counterpart. */ +#ifdef __GLIBC__ +# define LANGINFO(X, Y) {#X, X, Y, _NL_W ## X} +# define LANGINFO_NW(X, Y) {#X, X, Y, 0} +#else +# define LANGINFO(X, Y) {#X, X, Y} +# define LANGINFO_NW(X, Y) {#X, X, Y} +#endif static struct langinfo_constant{ const char *name; int value; int category; +#ifdef __GLIBC__ + nl_item wide; /* wide _NL_W* counterpart, or 0 if none */ +#endif } langinfo_constants[] = { /* These constants should exist on any langinfo implementation */ @@ -613,8 +625,8 @@ static struct langinfo_constant{ #ifdef RADIXCHAR /* The following are not available with glibc 2.0 */ - LANGINFO(RADIXCHAR, LC_NUMERIC), - LANGINFO(THOUSEP, LC_NUMERIC), + LANGINFO_NW(RADIXCHAR, LC_NUMERIC), + LANGINFO_NW(THOUSEP, LC_NUMERIC), /* YESSTR and NOSTR are deprecated in glibc, since they are a special case of message translation, which should be rather done using gettext. So we don't expose it to Python in the @@ -622,7 +634,7 @@ static struct langinfo_constant{ LANGINFO(YESSTR, LC_MESSAGES), LANGINFO(NOSTR, LC_MESSAGES), */ - LANGINFO(CRNCYSTR, LC_MONETARY), + LANGINFO_NW(CRNCYSTR, LC_MONETARY), #endif LANGINFO(D_T_FMT, LC_TIME), @@ -636,13 +648,13 @@ static struct langinfo_constant{ a few of the others. Solution: ifdef-test them all. */ #ifdef CODESET - LANGINFO(CODESET, LC_CTYPE), + LANGINFO_NW(CODESET, LC_CTYPE), #endif #ifdef T_FMT_AMPM LANGINFO(T_FMT_AMPM, LC_TIME), #endif #ifdef ERA - LANGINFO(ERA, LC_TIME), + LANGINFO_NW(ERA, LC_TIME), #endif #ifdef ERA_D_FMT LANGINFO(ERA_D_FMT, LC_TIME), @@ -657,10 +669,10 @@ static struct langinfo_constant{ LANGINFO(ALT_DIGITS, LC_TIME), #endif #ifdef YESEXPR - LANGINFO(YESEXPR, LC_MESSAGES), + LANGINFO_NW(YESEXPR, LC_MESSAGES), #endif #ifdef NOEXPR - LANGINFO(NOEXPR, LC_MESSAGES), + LANGINFO_NW(NOEXPR, LC_MESSAGES), #endif #ifdef _DATE_FMT /* This is not available in all glibc versions that have CODESET. */ @@ -709,15 +721,14 @@ restore_locale(char *oldloc) } #ifdef __GLIBC__ -#if defined(ALT_DIGITS) || defined(ERA) static PyObject * -decode_strings(const char *result, size_t max_count) +decode_strings(const char *result) { /* Convert a sequence of NUL-separated C strings to a Python string * containing semicolon separated items. */ size_t i = 0; size_t count = 0; - for (; count < max_count && result[i]; count++) { + for (; result[i]; count++) { i += strlen(result + i) + 1; } char *buf = PyMem_Malloc(i); @@ -737,7 +748,35 @@ decode_strings(const char *result, size_t max_count) return pyresult; } #endif -#endif + +#ifdef __GLIBC__ +static PyObject * +decode_wide_strings(const wchar_t *result, size_t max_count) +{ + /* Wide-character counterpart of decode_strings(): convert a sequence of + * NUL-separated wchar_t strings to a str with semicolon-separated items. */ + size_t i = 0; + size_t count = 0; + for (; count < max_count && result[i]; count++) { + i += wcslen(result + i) + 1; + } + wchar_t *buf = PyMem_New(wchar_t, i); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + memcpy(buf, result, i * sizeof(wchar_t)); + /* Replace all NULs with semicolons. */ + i = 0; + while (--count) { + i += wcslen(buf + i); + buf[i++] = L';'; + } + PyObject *pyresult = PyUnicode_FromWideChar(buf, -1); + PyMem_Free(buf); + return pyresult; +} +#endif /* __GLIBC__ */ /*[clinic input] _locale.nl_langinfo @@ -758,6 +797,22 @@ _locale_nl_langinfo_impl(PyObject *module, int item) crash PyUnicode_FromString. */ for (i = 0; langinfo_constants[i].name; i++) { if (langinfo_constants[i].value == item) { +#ifdef __GLIBC__ + /* Prefer the wide variant: it is decoded independently of the + LC_CTYPE encoding. */ + nl_item wide = langinfo_constants[i].wide; + if (wide) { + const wchar_t *wresult = (const wchar_t *)nl_langinfo(wide); + if (wresult == NULL) { + wresult = L""; + } + /* ALT_DIGITS is a sequence of NUL-separated strings. */ + if (item == ALT_DIGITS && *wresult) { + return decode_wide_strings(wresult, 100); + } + return PyUnicode_FromWideChar(wresult, -1); + } +#endif /* Check NULL as a workaround for GNU libc's returning NULL instead of an empty string for nl_langinfo(ERA). */ const char *result = nl_langinfo(item); @@ -766,13 +821,8 @@ _locale_nl_langinfo_impl(PyObject *module, int item) if (langinfo_constants[i].category != LC_CTYPE && *result && ( #ifdef __GLIBC__ - // gh-133740: Always change the locale for ALT_DIGITS and ERA -# ifdef ALT_DIGITS - item == ALT_DIGITS || -# endif -# ifdef ERA + // gh-133740: Always change the locale for ERA item == ERA || -# endif #endif !is_all_ascii(result)) && change_locale(langinfo_constants[i].category, &oldloc) < 0) @@ -784,18 +834,10 @@ _locale_nl_langinfo_impl(PyObject *module, int item) /* According to the POSIX specification the result must be * a sequence of semicolon-separated strings. * But in Glibc they are NUL-separated. */ -#ifdef ALT_DIGITS - if (item == ALT_DIGITS && *result) { - pyresult = decode_strings(result, 100); - } - else -#endif -#ifdef ERA if (item == ERA && *result) { - pyresult = decode_strings(result, SIZE_MAX); + pyresult = decode_strings(result); } else -#endif #endif { pyresult = PyUnicode_DecodeLocale(result, NULL);