From 6f44a1beab80fcfd1a6696868339a2364e887e28 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 5 Jul 2026 23:37:13 +0300 Subject: [PATCH 1/4] gh-66788: Add the imap4-utf-7 codec Implement the modified UTF-7 encoding used for international IMAP4 mailbox names (RFC 3501, section 5.1.3). It differs from UTF-7: "&" is the shift character ("&-" encodes a literal "&"), "," replaces "/" in the Base64 alphabet, and all non-printable-ASCII characters are Base64-encoded. Co-Authored-By: Claude Fable 5 --- Doc/library/codecs.rst | 8 ++ Doc/whatsnew/3.16.rst | 8 ++ Lib/encodings/imap4_utf_7.py | 126 ++++++++++++++++++ Lib/test/test_codecs.py | 67 ++++++++++ ...6-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst | 2 + 5 files changed, 211 insertions(+) create mode 100644 Lib/encodings/imap4_utf_7.py create mode 100644 Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 99fcf35aa893e42..de9e696563c5c13 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1384,6 +1384,14 @@ encodings. | | | Only ``errors='strict'`` | | | | is supported. | +--------------------+---------+---------------------------+ +| imap4-utf-7 | | Modified UTF-7 encoding | +| | | of :rfc:`3501` for IMAP4 | +| | | mailbox names. Only | +| | | ``errors='strict'`` is | +| | | supported. | +| | | | +| | | .. versionadded:: next | ++--------------------+---------+---------------------------+ | mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | | | | ANSI codepage (CP_ACP). | diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e8c530e19d2b53b..4b74dc4850005e5 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -214,6 +214,14 @@ curses wide-character support. (Contributed by Serhiy Storchaka in :gh:`133031`.) +encodings +--------- + +* Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding + used for international IMAP4 mailbox names (:rfc:`3501`). + (Contributed by Serhiy Storchaka in :gh:`66788`.) + + gzip ---- diff --git a/Lib/encodings/imap4_utf_7.py b/Lib/encodings/imap4_utf_7.py new file mode 100644 index 000000000000000..f0b9df85260e446 --- /dev/null +++ b/Lib/encodings/imap4_utf_7.py @@ -0,0 +1,126 @@ +""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names, +as specified in RFC 3501, section 5.1.3. + +It differs from UTF-7 (RFC 2152) as follows: + +* "&" (not "+") is the shift character introducing a Base64 sequence, + and "&-" encodes a literal "&"; +* "," is used instead of "/" in the modified Base64 alphabet; +* only printable US-ASCII characters (except "&") may be represented + directly, so all other characters, including other controls, are + Base64-encoded. +""" + +import binascii +import codecs +from base64 import b64encode, b64decode + +### Codec APIs + +def imap4_utf_7_encode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + res = bytearray() + start = 0 # start of the current run + b64run = False # is input[start:i] a Base64 run? + def flush(end): + if start < end: + if b64run: + b64 = b64encode(input[start:end].encode('utf-16-be'), + altchars=b'+,', padded=False) + res.extend(b'&' + b64 + b'-') + else: + res.extend(input[start:end].encode('ascii')) + for i, ch in enumerate(input): + if ch == '&': + flush(i) + res.extend(b'&-') + start = i + 1 + b64run = False + elif ' ' <= ch <= '~': # printable ASCII, represented directly + if b64run: + flush(i) + start = i + b64run = False + else: # everything else is Base64-encoded + if not b64run: + flush(i) + start = i + b64run = True + flush(len(input)) + return res.take_bytes(), len(input) + +def imap4_utf_7_decode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + input = bytes(input) + res = [] + start = 0 # start of the current direct ASCII run + i = 0 + n = len(input) + def flush(end): + if start < end: + res.append(input[start:end].decode('ascii')) + while i < n: + c = input[i] + if c == b'&'[0]: + flush(i) + j = input.find(b'-', i + 1) + if j < 0: + raise UnicodeDecodeError('imap4-utf-7', input, i, n, + 'unterminated shift sequence') + if j == i + 1: # '&-' + res.append('&') + else: + b64 = input[i + 1:j] + try: + data = b64decode(b64, altchars=b'+,', + validate=True, padded=False) + except binascii.Error: + data = b'' + if not data or len(data) % 2: + raise UnicodeDecodeError('imap4-utf-7', input, i, j + 1, + 'invalid shift sequence') + res.append(data.decode('utf-16-be')) + i = j + 1 + start = i + elif b' '[0] <= c <= b'~'[0]: + i += 1 + else: + raise UnicodeDecodeError('imap4-utf-7', input, i, i + 1, + 'unexpected byte') + flush(n) + return ''.join(res), len(input) + +class Codec(codecs.Codec): + def encode(self, input, errors='strict'): + return imap4_utf_7_encode(input, errors) + def decode(self, input, errors='strict'): + return imap4_utf_7_decode(input, errors) + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return imap4_utf_7_encode(input, self.errors)[0] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return imap4_utf_7_decode(input, self.errors)[0] + +class StreamWriter(Codec, codecs.StreamWriter): + pass + +class StreamReader(Codec, codecs.StreamReader): + pass + +### encodings module API + +def getregentry(): + return codecs.CodecInfo( + name='imap4-utf-7', + encode=Codec().encode, + decode=Codec().decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 8fdd08df9e4f46a..5b911fddd63fd28 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1401,6 +1401,73 @@ def test_decode_invalid(self): self.assertEqual(puny.decode("punycode", errors), expected) +imap4_utf_7_testcases = [ + # (unicode, modified UTF-7) + ('', b''), + ('INBOX', b'INBOX'), + # Printable US-ASCII represents itself. + ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'), + # "&" is escaped as "&-". + ('&', b'&-'), + ('&&', b'&-&-'), + ('A&B', b'A&-B'), + # RFC 3501 section 5.1.3 example. + ('~peter/mail/台北/日本語', + b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'), + # Non-printable ASCII (including TAB) is Base64-encoded, not direct. + ('a\tb', b'a&AAk-b'), + ('\x00', b'&AAA-'), + ('Entw\xfcrfe', b'Entw&APw-rfe'), + ('☃', b'&JgM-'), # snowman + ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair) + ('Sent &\N{DELETE}', b'Sent &-&AH8-'), +] + +class Imap4Utf7Test(unittest.TestCase): + def test_encode(self): + for uni, encoded in imap4_utf_7_testcases: + with self.subTest(uni=uni): + self.assertEqual(uni.encode('imap4-utf-7'), encoded) + + def test_decode(self): + for uni, encoded in imap4_utf_7_testcases: + with self.subTest(encoded=encoded): + self.assertEqual(encoded.decode('imap4-utf-7'), uni) + + def test_decode_invalid(self): + # position of the first offending byte in each case + testcases = [ + (b'&AAk', 0), # unterminated shift sequence + (b'&Jgg', 0), # unterminated shift sequence + (b'&AB-', 0), # Base64 length not a multiple of a code unit + (b'&@@@-', 0), # invalid Base64 + (b'&====-', 0), # invalid Base64 + (b'a\x80b', 1), # 8-bit byte outside a shift sequence + (b'a\x1fb', 1), # control byte outside a shift sequence + ] + for encoded, start in testcases: + with self.subTest(encoded=encoded): + with self.assertRaises(UnicodeDecodeError) as cm: + encoded.decode('imap4-utf-7') + self.assertEqual(cm.exception.encoding, 'imap4-utf-7') + self.assertEqual(cm.exception.start, start) + + def test_encode_lone_surrogate(self): + with self.assertRaises(UnicodeEncodeError): + '\ud800'.encode('imap4-utf-7') + + def test_only_strict_errors(self): + with self.assertRaises(UnicodeError): + 'x'.encode('imap4-utf-7', 'replace') + with self.assertRaises(UnicodeError): + b'x'.decode('imap4-utf-7', 'ignore') + + def test_stateless(self): + # The codec is registered and exposes the standard interface. + info = codecs.lookup('imap4-utf-7') + self.assertEqual(info.name, 'imap4-utf-7') + + # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ # 3.1 Map to nothing. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst new file mode 100644 index 000000000000000..b46adc17bd3e675 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst @@ -0,0 +1,2 @@ +Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding +used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3). From 083de621cd9c715d63401aa48df0be0f9c183370 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 Jul 2026 19:21:59 +0300 Subject: [PATCH 2/4] gh-66788: Rename the codec to utf-7-imap and add aliases Rename the modified-UTF-7 codec from imap4-utf-7 to utf-7-imap and register the aliases csUTF7IMAP and mUTF-7. Co-Authored-By: Claude Opus 4.8 --- Doc/library/codecs.rst | 16 ++++++------ Doc/whatsnew/3.16.rst | 2 +- Lib/encodings/aliases.py | 4 +++ .../{imap4_utf_7.py => utf_7_imap.py} | 20 +++++++------- Lib/test/test_codecs.py | 26 +++++++++---------- ...6-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst | 2 +- 6 files changed, 37 insertions(+), 33 deletions(-) rename Lib/encodings/{imap4_utf_7.py => utf_7_imap.py} (87%) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index de9e696563c5c13..b871942837d7c9b 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1384,14 +1384,6 @@ encodings. | | | Only ``errors='strict'`` | | | | is supported. | +--------------------+---------+---------------------------+ -| imap4-utf-7 | | Modified UTF-7 encoding | -| | | of :rfc:`3501` for IMAP4 | -| | | mailbox names. Only | -| | | ``errors='strict'`` is | -| | | supported. | -| | | | -| | | .. versionadded:: next | -+--------------------+---------+---------------------------+ | mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | | | | ANSI codepage (CP_ACP). | @@ -1446,6 +1438,14 @@ encodings. | | | code actually uses UTF-8 | | | | by default. | +--------------------+---------+---------------------------+ +| utf-7-imap | mUTF-7 | Modified UTF-7 encoding | +| | | of :rfc:`3501` for IMAP4 | +| | | mailbox names. Only | +| | | ``errors='strict'`` is | +| | | supported. | +| | | | +| | | .. versionadded:: next | ++--------------------+---------+---------------------------+ .. versionchanged:: 3.8 "unicode_internal" codec is removed. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4b74dc4850005e5..c0a493d9e84bc74 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -217,7 +217,7 @@ curses encodings --------- -* Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding +* Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding used for international IMAP4 mailbox names (:rfc:`3501`). (Contributed by Serhiy Storchaka in :gh:`66788`.) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index ef51168d755ba98..de26353aecaaac2 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -600,6 +600,10 @@ 'utf7' : 'utf_7', 'unicode_1_1_utf_7' : 'utf_7', + # utf_7_imap codec + 'csutf7imap' : 'utf_7_imap', + 'mutf_7' : 'utf_7_imap', + # utf_8 codec 'csutf8' : 'utf_8', 'u8' : 'utf_8', diff --git a/Lib/encodings/imap4_utf_7.py b/Lib/encodings/utf_7_imap.py similarity index 87% rename from Lib/encodings/imap4_utf_7.py rename to Lib/encodings/utf_7_imap.py index f0b9df85260e446..f49a94d1db4a6d4 100644 --- a/Lib/encodings/imap4_utf_7.py +++ b/Lib/encodings/utf_7_imap.py @@ -17,7 +17,7 @@ ### Codec APIs -def imap4_utf_7_encode(input, errors='strict'): +def utf_7_imap_encode(input, errors='strict'): if errors != 'strict': raise UnicodeError(f"Unsupported error handling: {errors}") res = bytearray() @@ -50,7 +50,7 @@ def flush(end): flush(len(input)) return res.take_bytes(), len(input) -def imap4_utf_7_decode(input, errors='strict'): +def utf_7_imap_decode(input, errors='strict'): if errors != 'strict': raise UnicodeError(f"Unsupported error handling: {errors}") input = bytes(input) @@ -67,7 +67,7 @@ def flush(end): flush(i) j = input.find(b'-', i + 1) if j < 0: - raise UnicodeDecodeError('imap4-utf-7', input, i, n, + raise UnicodeDecodeError('utf-7-imap', input, i, n, 'unterminated shift sequence') if j == i + 1: # '&-' res.append('&') @@ -79,7 +79,7 @@ def flush(end): except binascii.Error: data = b'' if not data or len(data) % 2: - raise UnicodeDecodeError('imap4-utf-7', input, i, j + 1, + raise UnicodeDecodeError('utf-7-imap', input, i, j + 1, 'invalid shift sequence') res.append(data.decode('utf-16-be')) i = j + 1 @@ -87,24 +87,24 @@ def flush(end): elif b' '[0] <= c <= b'~'[0]: i += 1 else: - raise UnicodeDecodeError('imap4-utf-7', input, i, i + 1, + raise UnicodeDecodeError('utf-7-imap', input, i, i + 1, 'unexpected byte') flush(n) return ''.join(res), len(input) class Codec(codecs.Codec): def encode(self, input, errors='strict'): - return imap4_utf_7_encode(input, errors) + return utf_7_imap_encode(input, errors) def decode(self, input, errors='strict'): - return imap4_utf_7_decode(input, errors) + return utf_7_imap_decode(input, errors) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return imap4_utf_7_encode(input, self.errors)[0] + return utf_7_imap_encode(input, self.errors)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return imap4_utf_7_decode(input, self.errors)[0] + return utf_7_imap_decode(input, self.errors)[0] class StreamWriter(Codec, codecs.StreamWriter): pass @@ -116,7 +116,7 @@ class StreamReader(Codec, codecs.StreamReader): def getregentry(): return codecs.CodecInfo( - name='imap4-utf-7', + name='utf-7-imap', encode=Codec().encode, decode=Codec().decode, incrementalencoder=IncrementalEncoder, diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 5b911fddd63fd28..cf5824493425526 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1401,7 +1401,7 @@ def test_decode_invalid(self): self.assertEqual(puny.decode("punycode", errors), expected) -imap4_utf_7_testcases = [ +utf_7_imap_testcases = [ # (unicode, modified UTF-7) ('', b''), ('INBOX', b'INBOX'), @@ -1423,16 +1423,16 @@ def test_decode_invalid(self): ('Sent &\N{DELETE}', b'Sent &-&AH8-'), ] -class Imap4Utf7Test(unittest.TestCase): +class Utf7ImapTest(unittest.TestCase): def test_encode(self): - for uni, encoded in imap4_utf_7_testcases: + for uni, encoded in utf_7_imap_testcases: with self.subTest(uni=uni): - self.assertEqual(uni.encode('imap4-utf-7'), encoded) + self.assertEqual(uni.encode('utf-7-imap'), encoded) def test_decode(self): - for uni, encoded in imap4_utf_7_testcases: + for uni, encoded in utf_7_imap_testcases: with self.subTest(encoded=encoded): - self.assertEqual(encoded.decode('imap4-utf-7'), uni) + self.assertEqual(encoded.decode('utf-7-imap'), uni) def test_decode_invalid(self): # position of the first offending byte in each case @@ -1448,24 +1448,24 @@ def test_decode_invalid(self): for encoded, start in testcases: with self.subTest(encoded=encoded): with self.assertRaises(UnicodeDecodeError) as cm: - encoded.decode('imap4-utf-7') - self.assertEqual(cm.exception.encoding, 'imap4-utf-7') + encoded.decode('utf-7-imap') + self.assertEqual(cm.exception.encoding, 'utf-7-imap') self.assertEqual(cm.exception.start, start) def test_encode_lone_surrogate(self): with self.assertRaises(UnicodeEncodeError): - '\ud800'.encode('imap4-utf-7') + '\ud800'.encode('utf-7-imap') def test_only_strict_errors(self): with self.assertRaises(UnicodeError): - 'x'.encode('imap4-utf-7', 'replace') + 'x'.encode('utf-7-imap', 'replace') with self.assertRaises(UnicodeError): - b'x'.decode('imap4-utf-7', 'ignore') + b'x'.decode('utf-7-imap', 'ignore') def test_stateless(self): # The codec is registered and exposes the standard interface. - info = codecs.lookup('imap4-utf-7') - self.assertEqual(info.name, 'imap4-utf-7') + info = codecs.lookup('utf-7-imap') + self.assertEqual(info.name, 'utf-7-imap') # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html diff --git a/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst index b46adc17bd3e675..29bd36ebd6295fe 100644 --- a/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst +++ b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst @@ -1,2 +1,2 @@ -Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding +Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3). From 09ed62fac5a11744ffe4300bb7233403f3a64153 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 Jul 2026 20:00:41 +0300 Subject: [PATCH 3/4] gh-66788: Use test.support.subTests in utf-7-imap codec tests Convert the parametrized utf-7-imap codec tests to the subTests() decorator. Add round-trip cases for "+", "+-" (literal, unlike in UTF-7) and a "," in the Base64 run, and replace the duplicate decode-error cases with a "&" at the end of the input and an unterminated but otherwise valid Base64 sequence. Co-Authored-By: Claude Opus 4.8 --- Lib/test/test_codecs.py | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index cf5824493425526..6f68a1a89ea28d7 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1411,6 +1411,9 @@ def test_decode_invalid(self): ('&', b'&-'), ('&&', b'&-&-'), ('A&B', b'A&-B'), + # "+" and "+-" are literal, unlike in UTF-7 where "+" is the shift character. + ('+', b'+'), + ('+-', b'+-'), # RFC 3501 section 5.1.3 example. ('~peter/mail/台北/日本語', b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'), @@ -1418,39 +1421,35 @@ def test_decode_invalid(self): ('a\tb', b'a&AAk-b'), ('\x00', b'&AAA-'), ('Entw\xfcrfe', b'Entw&APw-rfe'), + ('ϰ', b'&A,A-'), # "," in the modified Base64 alphabet ('☃', b'&JgM-'), # snowman ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair) ('Sent &\N{DELETE}', b'Sent &-&AH8-'), ] class Utf7ImapTest(unittest.TestCase): - def test_encode(self): - for uni, encoded in utf_7_imap_testcases: - with self.subTest(uni=uni): - self.assertEqual(uni.encode('utf-7-imap'), encoded) - - def test_decode(self): - for uni, encoded in utf_7_imap_testcases: - with self.subTest(encoded=encoded): - self.assertEqual(encoded.decode('utf-7-imap'), uni) - - def test_decode_invalid(self): - # position of the first offending byte in each case - testcases = [ - (b'&AAk', 0), # unterminated shift sequence - (b'&Jgg', 0), # unterminated shift sequence - (b'&AB-', 0), # Base64 length not a multiple of a code unit - (b'&@@@-', 0), # invalid Base64 - (b'&====-', 0), # invalid Base64 - (b'a\x80b', 1), # 8-bit byte outside a shift sequence - (b'a\x1fb', 1), # control byte outside a shift sequence - ] - for encoded, start in testcases: - with self.subTest(encoded=encoded): - with self.assertRaises(UnicodeDecodeError) as cm: - encoded.decode('utf-7-imap') - self.assertEqual(cm.exception.encoding, 'utf-7-imap') - self.assertEqual(cm.exception.start, start) + @support.subTests('uni,encoded', utf_7_imap_testcases) + def test_encode(self, uni, encoded): + self.assertEqual(uni.encode('utf-7-imap'), encoded) + + @support.subTests('uni,encoded', utf_7_imap_testcases) + def test_decode(self, uni, encoded): + self.assertEqual(encoded.decode('utf-7-imap'), uni) + + # 'start' is the position of the first offending byte in each case. + @support.subTests('encoded,start', [ + (b'x&', 1), # "&" just before the end, unterminated + (b'&AAAA', 0), # unterminated, though the Base64 is valid + (b'&AB-', 0), # Base64 length not a multiple of a code unit + (b'&@@@-', 0), # invalid Base64 + (b'a\x80b', 1), # 8-bit byte outside a shift sequence + (b'a\x1fb', 1), # control byte outside a shift sequence + ]) + def test_decode_invalid(self, encoded, start): + with self.assertRaises(UnicodeDecodeError) as cm: + encoded.decode('utf-7-imap') + self.assertEqual(cm.exception.encoding, 'utf-7-imap') + self.assertEqual(cm.exception.start, start) def test_encode_lone_surrogate(self): with self.assertRaises(UnicodeEncodeError): From 6dd962fd0eae514eb89b9c21785b242b91d4821b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 Jul 2026 20:12:07 +0300 Subject: [PATCH 4/4] gh-66788: Strictly enforce the modified Base64 alphabet in utf-7-imap Decode the modified Base64 with binascii.a2b_base64() and the modified alphabet directly, instead of base64.b64decode() with altchars. With strict_mode the "/" character (used by standard Base64 but not by the modified alphabet, which uses ",") is now rejected rather than accepted with a DeprecationWarning. Encoding likewise uses binascii.b2a_base64(). Co-Authored-By: Claude Opus 4.8 --- Lib/encodings/utf_7_imap.py | 14 +++++++++----- Lib/test/test_codecs.py | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/encodings/utf_7_imap.py b/Lib/encodings/utf_7_imap.py index f49a94d1db4a6d4..159c8591cc1ff24 100644 --- a/Lib/encodings/utf_7_imap.py +++ b/Lib/encodings/utf_7_imap.py @@ -13,7 +13,10 @@ import binascii import codecs -from base64 import b64encode, b64decode + +# The modified Base64 alphabet of RFC 3501: standard Base64 but with "," in +# place of "/". +_alphabet = binascii.BASE64_ALPHABET[:-1] + b',' ### Codec APIs @@ -26,8 +29,9 @@ def utf_7_imap_encode(input, errors='strict'): def flush(end): if start < end: if b64run: - b64 = b64encode(input[start:end].encode('utf-16-be'), - altchars=b'+,', padded=False) + b64 = binascii.b2a_base64(input[start:end].encode('utf-16-be'), + alphabet=_alphabet, padded=False, + newline=False) res.extend(b'&' + b64 + b'-') else: res.extend(input[start:end].encode('ascii')) @@ -74,8 +78,8 @@ def flush(end): else: b64 = input[i + 1:j] try: - data = b64decode(b64, altchars=b'+,', - validate=True, padded=False) + data = binascii.a2b_base64(b64, alphabet=_alphabet, + strict_mode=True, padded=False) except binascii.Error: data = b'' if not data or len(data) % 2: diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 6f68a1a89ea28d7..6f47db25271ffee 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1421,7 +1421,7 @@ def test_decode_invalid(self): ('a\tb', b'a&AAk-b'), ('\x00', b'&AAA-'), ('Entw\xfcrfe', b'Entw&APw-rfe'), - ('ϰ', b'&A,A-'), # "," in the modified Base64 alphabet + ('ϰ', b'&A,A-'), # "," in the Base64 alphabet ("&A/A-" is invalid) ('☃', b'&JgM-'), # snowman ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair) ('Sent &\N{DELETE}', b'Sent &-&AH8-'), @@ -1441,6 +1441,7 @@ def test_decode(self, uni, encoded): (b'x&', 1), # "&" just before the end, unterminated (b'&AAAA', 0), # unterminated, though the Base64 is valid (b'&AB-', 0), # Base64 length not a multiple of a code unit + (b'&A/A-', 0), # "/" not in the alphabet ("&A,A-" is valid) (b'&@@@-', 0), # invalid Base64 (b'a\x80b', 1), # 8-bit byte outside a shift sequence (b'a\x1fb', 1), # control byte outside a shift sequence