diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 99fcf35aa893e42..b871942837d7c9b 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1438,6 +1438,14 @@ encodings. | | | code actually uses UTF-8 | | | | by default. | +--------------------+---------+---------------------------+ +| utf-7-imap | mUTF-7 | Modified UTF-7 encoding | +| | | of :rfc:`3501` for IMAP4 | +| | | mailbox names. Only | +| | | ``errors='strict'`` is | +| | | supported. | +| | | | +| | | .. versionadded:: next | ++--------------------+---------+---------------------------+ .. versionchanged:: 3.8 "unicode_internal" codec is removed. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e8c530e19d2b53b..c0a493d9e84bc74 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -214,6 +214,14 @@ curses wide-character support. (Contributed by Serhiy Storchaka in :gh:`133031`.) +encodings +--------- + +* Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding + used for international IMAP4 mailbox names (:rfc:`3501`). + (Contributed by Serhiy Storchaka in :gh:`66788`.) + + gzip ---- diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index ef51168d755ba98..de26353aecaaac2 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -600,6 +600,10 @@ 'utf7' : 'utf_7', 'unicode_1_1_utf_7' : 'utf_7', + # utf_7_imap codec + 'csutf7imap' : 'utf_7_imap', + 'mutf_7' : 'utf_7_imap', + # utf_8 codec 'csutf8' : 'utf_8', 'u8' : 'utf_8', diff --git a/Lib/encodings/utf_7_imap.py b/Lib/encodings/utf_7_imap.py new file mode 100644 index 000000000000000..159c8591cc1ff24 --- /dev/null +++ b/Lib/encodings/utf_7_imap.py @@ -0,0 +1,130 @@ +""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names, +as specified in RFC 3501, section 5.1.3. + +It differs from UTF-7 (RFC 2152) as follows: + +* "&" (not "+") is the shift character introducing a Base64 sequence, + and "&-" encodes a literal "&"; +* "," is used instead of "/" in the modified Base64 alphabet; +* only printable US-ASCII characters (except "&") may be represented + directly, so all other characters, including other controls, are + Base64-encoded. +""" + +import binascii +import codecs + +# The modified Base64 alphabet of RFC 3501: standard Base64 but with "," in +# place of "/". +_alphabet = binascii.BASE64_ALPHABET[:-1] + b',' + +### Codec APIs + +def utf_7_imap_encode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + res = bytearray() + start = 0 # start of the current run + b64run = False # is input[start:i] a Base64 run? + def flush(end): + if start < end: + if b64run: + b64 = binascii.b2a_base64(input[start:end].encode('utf-16-be'), + alphabet=_alphabet, padded=False, + newline=False) + res.extend(b'&' + b64 + b'-') + else: + res.extend(input[start:end].encode('ascii')) + for i, ch in enumerate(input): + if ch == '&': + flush(i) + res.extend(b'&-') + start = i + 1 + b64run = False + elif ' ' <= ch <= '~': # printable ASCII, represented directly + if b64run: + flush(i) + start = i + b64run = False + else: # everything else is Base64-encoded + if not b64run: + flush(i) + start = i + b64run = True + flush(len(input)) + return res.take_bytes(), len(input) + +def utf_7_imap_decode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + input = bytes(input) + res = [] + start = 0 # start of the current direct ASCII run + i = 0 + n = len(input) + def flush(end): + if start < end: + res.append(input[start:end].decode('ascii')) + while i < n: + c = input[i] + if c == b'&'[0]: + flush(i) + j = input.find(b'-', i + 1) + if j < 0: + raise UnicodeDecodeError('utf-7-imap', input, i, n, + 'unterminated shift sequence') + if j == i + 1: # '&-' + res.append('&') + else: + b64 = input[i + 1:j] + try: + data = binascii.a2b_base64(b64, alphabet=_alphabet, + strict_mode=True, padded=False) + except binascii.Error: + data = b'' + if not data or len(data) % 2: + raise UnicodeDecodeError('utf-7-imap', input, i, j + 1, + 'invalid shift sequence') + res.append(data.decode('utf-16-be')) + i = j + 1 + start = i + elif b' '[0] <= c <= b'~'[0]: + i += 1 + else: + raise UnicodeDecodeError('utf-7-imap', input, i, i + 1, + 'unexpected byte') + flush(n) + return ''.join(res), len(input) + +class Codec(codecs.Codec): + def encode(self, input, errors='strict'): + return utf_7_imap_encode(input, errors) + def decode(self, input, errors='strict'): + return utf_7_imap_decode(input, errors) + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return utf_7_imap_encode(input, self.errors)[0] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return utf_7_imap_decode(input, self.errors)[0] + +class StreamWriter(Codec, codecs.StreamWriter): + pass + +class StreamReader(Codec, codecs.StreamReader): + pass + +### encodings module API + +def getregentry(): + return codecs.CodecInfo( + name='utf-7-imap', + encode=Codec().encode, + decode=Codec().decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 8fdd08df9e4f46a..6f47db25271ffee 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1401,6 +1401,73 @@ def test_decode_invalid(self): self.assertEqual(puny.decode("punycode", errors), expected) +utf_7_imap_testcases = [ + # (unicode, modified UTF-7) + ('', b''), + ('INBOX', b'INBOX'), + # Printable US-ASCII represents itself. + ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'), + # "&" is escaped as "&-". + ('&', b'&-'), + ('&&', b'&-&-'), + ('A&B', b'A&-B'), + # "+" and "+-" are literal, unlike in UTF-7 where "+" is the shift character. + ('+', b'+'), + ('+-', b'+-'), + # RFC 3501 section 5.1.3 example. + ('~peter/mail/台北/日本語', + b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'), + # Non-printable ASCII (including TAB) is Base64-encoded, not direct. + ('a\tb', b'a&AAk-b'), + ('\x00', b'&AAA-'), + ('Entw\xfcrfe', b'Entw&APw-rfe'), + ('ϰ', b'&A,A-'), # "," in the Base64 alphabet ("&A/A-" is invalid) + ('☃', b'&JgM-'), # snowman + ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair) + ('Sent &\N{DELETE}', b'Sent &-&AH8-'), +] + +class Utf7ImapTest(unittest.TestCase): + @support.subTests('uni,encoded', utf_7_imap_testcases) + def test_encode(self, uni, encoded): + self.assertEqual(uni.encode('utf-7-imap'), encoded) + + @support.subTests('uni,encoded', utf_7_imap_testcases) + def test_decode(self, uni, encoded): + self.assertEqual(encoded.decode('utf-7-imap'), uni) + + # 'start' is the position of the first offending byte in each case. + @support.subTests('encoded,start', [ + (b'x&', 1), # "&" just before the end, unterminated + (b'&AAAA', 0), # unterminated, though the Base64 is valid + (b'&AB-', 0), # Base64 length not a multiple of a code unit + (b'&A/A-', 0), # "/" not in the alphabet ("&A,A-" is valid) + (b'&@@@-', 0), # invalid Base64 + (b'a\x80b', 1), # 8-bit byte outside a shift sequence + (b'a\x1fb', 1), # control byte outside a shift sequence + ]) + def test_decode_invalid(self, encoded, start): + with self.assertRaises(UnicodeDecodeError) as cm: + encoded.decode('utf-7-imap') + self.assertEqual(cm.exception.encoding, 'utf-7-imap') + self.assertEqual(cm.exception.start, start) + + def test_encode_lone_surrogate(self): + with self.assertRaises(UnicodeEncodeError): + '\ud800'.encode('utf-7-imap') + + def test_only_strict_errors(self): + with self.assertRaises(UnicodeError): + 'x'.encode('utf-7-imap', 'replace') + with self.assertRaises(UnicodeError): + b'x'.decode('utf-7-imap', 'ignore') + + def test_stateless(self): + # The codec is registered and exposes the standard interface. + info = codecs.lookup('utf-7-imap') + self.assertEqual(info.name, 'utf-7-imap') + + # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ # 3.1 Map to nothing. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst new file mode 100644 index 000000000000000..29bd36ebd6295fe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst @@ -0,0 +1,2 @@ +Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding +used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3).