Skip to content

Commit 46d88a1

Browse files
maxkingwarsaw
maxking
authored and
warsaw
committed
bpo-35805: Add parser for Message-ID email header. (pythonGH-13397)
* bpo-35805: Add parser for Message-ID header. This parser is based on the definition of Identification Fields from RFC 5322 Sec 3.6.4. This should also prevent folding of Message-ID header using RFC 2047 encoded words and hence fix bpo-35805. * Prevent folding of non-ascii message-id headers. * Add fold method to MsgID token to prevent folding.
1 parent bc6469f commit 46d88a1

File tree

6 files changed

+257
-28
lines changed

6 files changed

+257
-28
lines changed

‎Doc/library/email.headerregistry.rst‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -321,19 +321,26 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1.
321321

322322
The default mappings are:
323323

324-
:subject: UniqueUnstructuredHeader
325-
:date: UniqueDateHeader
326-
:resent-date: DateHeader
327-
:orig-date: UniqueDateHeader
328-
:sender: UniqueSingleAddressHeader
329-
:resent-sender: SingleAddressHeader
330-
:to: UniqueAddressHeader
331-
:resent-to: AddressHeader
332-
:cc: UniqueAddressHeader
333-
:resent-cc: AddressHeader
334-
:from: UniqueAddressHeader
335-
:resent-from: AddressHeader
336-
:reply-to: UniqueAddressHeader
324+
:subject: UniqueUnstructuredHeader
325+
:date: UniqueDateHeader
326+
:resent-date: DateHeader
327+
:orig-date: UniqueDateHeader
328+
:sender: UniqueSingleAddressHeader
329+
:resent-sender: SingleAddressHeader
330+
:to: UniqueAddressHeader
331+
:resent-to: AddressHeader
332+
:cc: UniqueAddressHeader
333+
:resent-cc: AddressHeader
334+
:bcc: UniqueAddressHeader
335+
:resent-bcc: AddressHeader
336+
:from: UniqueAddressHeader
337+
:resent-from: AddressHeader
338+
:reply-to: UniqueAddressHeader
339+
:mime-version: MIMEVersionHeader
340+
:content-type: ContentTypeHeader
341+
:content-disposition: ContentDispositionHeader
342+
:content-transfer-encoding: ContentTransferEncodingHeader
343+
:message-id: MessageIDHeader
337344

338345
``HeaderRegistry`` has the following methods:
339346

‎Lib/email/_header_value_parser.py‎

Lines changed: 122 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -179,37 +179,30 @@ def comments(self):
179179

180180

181181
class UnstructuredTokenList(TokenList):
182-
183182
token_type = 'unstructured'
184183

185184

186185
class Phrase(TokenList):
187-
188186
token_type = 'phrase'
189187

190188
class Word(TokenList):
191-
192189
token_type = 'word'
193190

194191

195192
class CFWSList(WhiteSpaceTokenList):
196-
197193
token_type = 'cfws'
198194

199195

200196
class Atom(TokenList):
201-
202197
token_type = 'atom'
203198

204199

205200
class Token(TokenList):
206-
207201
token_type = 'token'
208202
encode_as_ew = False
209203

210204

211205
class EncodedWord(TokenList):
212-
213206
token_type = 'encoded-word'
214207
cte = None
215208
charset = None
@@ -496,16 +489,19 @@ def domain(self):
496489

497490

498491
class DotAtom(TokenList):
499-
500492
token_type = 'dot-atom'
501493

502494

503495
class DotAtomText(TokenList):
504-
505496
token_type = 'dot-atom-text'
506497
as_ew_allowed = True
507498

508499

500+
class NoFoldLiteral(TokenList):
501+
token_type = 'no-fold-literal'
502+
as_ew_allowed = False
503+
504+
509505
class AddrSpec(TokenList):
510506

511507
token_type = 'addr-spec'
@@ -809,35 +805,42 @@ def params(self):
809805

810806

811807
class ContentType(ParameterizedHeaderValue):
812-
813808
token_type = 'content-type'
814809
as_ew_allowed = False
815810
maintype = 'text'
816811
subtype = 'plain'
817812

818813

819814
class ContentDisposition(ParameterizedHeaderValue):
820-
821815
token_type = 'content-disposition'
822816
as_ew_allowed = False
823817
content_disposition = None
824818

825819

826820
class ContentTransferEncoding(TokenList):
827-
828821
token_type = 'content-transfer-encoding'
829822
as_ew_allowed = False
830823
cte = '7bit'
831824

832825

833826
class HeaderLabel(TokenList):
834-
835827
token_type = 'header-label'
836828
as_ew_allowed = False
837829

838830

839-
class Header(TokenList):
831+
class MsgID(TokenList):
832+
token_type = 'msg-id'
833+
as_ew_allowed = False
834+
835+
def fold(self, policy):
836+
# message-id tokens may not be folded.
837+
return str(self) + policy.linesep
838+
839+
class MessageID(MsgID):
840+
token_type = 'message-id'
840841

842+
843+
class Header(TokenList):
841844
token_type = 'header'
842845

843846

@@ -1583,7 +1586,7 @@ def get_addr_spec(value):
15831586
addr_spec.append(token)
15841587
if not value or value[0] != '@':
15851588
addr_spec.defects.append(errors.InvalidHeaderDefect(
1586-
"add-spec local part with no domain"))
1589+
"addr-spec local part with no domain"))
15871590
return addr_spec, value
15881591
addr_spec.append(ValueTerminal('@', 'address-at-symbol'))
15891592
token, value = get_domain(value[1:])
@@ -1968,6 +1971,110 @@ def get_address_list(value):
19681971
value = value[1:]
19691972
return address_list, value
19701973

1974+
1975+
def get_no_fold_literal(value):
1976+
""" no-fold-literal = "[" *dtext "]"
1977+
"""
1978+
no_fold_literal = NoFoldLiteral()
1979+
if not value:
1980+
raise errors.HeaderParseError(
1981+
"expected no-fold-literal but found '{}'".format(value))
1982+
if value[0] != '[':
1983+
raise errors.HeaderParseError(
1984+
"expected '[' at the start of no-fold-literal "
1985+
"but found '{}'".format(value))
1986+
no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start'))
1987+
value = value[1:]
1988+
token, value = get_dtext(value)
1989+
no_fold_literal.append(token)
1990+
if not value or value[0] != ']':
1991+
raise errors.HeaderParseError(
1992+
"expected ']' at the end of no-fold-literal "
1993+
"but found '{}'".format(value))
1994+
no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end'))
1995+
return no_fold_literal, value[1:]
1996+
1997+
def get_msg_id(value):
1998+
"""msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS]
1999+
id-left = dot-atom-text / obs-id-left
2000+
id-right = dot-atom-text / no-fold-literal / obs-id-right
2001+
no-fold-literal = "[" *dtext "]"
2002+
"""
2003+
msg_id = MsgID()
2004+
if value[0] in CFWS_LEADER:
2005+
token, value = get_cfws(value)
2006+
msg_id.append(token)
2007+
if not value or value[0] != '<':
2008+
raise errors.HeaderParseError(
2009+
"expected msg-id but found '{}'".format(value))
2010+
msg_id.append(ValueTerminal('<', 'msg-id-start'))
2011+
value = value[1:]
2012+
# Parse id-left.
2013+
try:
2014+
token, value = get_dot_atom_text(value)
2015+
except errors.HeaderParseError:
2016+
try:
2017+
# obs-id-left is same as local-part of add-spec.
2018+
token, value = get_obs_local_part(value)
2019+
msg_id.defects.append(errors.ObsoleteHeaderDefect(
2020+
"obsolete id-left in msg-id"))
2021+
except errors.HeaderParseError:
2022+
raise errors.HeaderParseError(
2023+
"expected dot-atom-text or obs-id-left"
2024+
" but found '{}'".format(value))
2025+
msg_id.append(token)
2026+
if not value or value[0] != '@':
2027+
msg_id.defects.append(errors.InvalidHeaderDefect(
2028+
"msg-id with no id-right"))
2029+
# Even though there is no id-right, if the local part
2030+
# ends with `>` let's just parse it too and return
2031+
# along with the defect.
2032+
if value and value[0] == '>':
2033+
msg_id.append(ValueTerminal('>', 'msg-id-end'))
2034+
value = value[1:]
2035+
return msg_id, value
2036+
msg_id.append(ValueTerminal('@', 'address-at-symbol'))
2037+
value = value[1:]
2038+
# Parse id-right.
2039+
try:
2040+
token, value = get_dot_atom_text(value)
2041+
except errors.HeaderParseError:
2042+
try:
2043+
token, value = get_no_fold_literal(value)
2044+
except errors.HeaderParseError as e:
2045+
try:
2046+
token, value = get_domain(value)
2047+
msg_id.defects.append(errors.ObsoleteHeaderDefect(
2048+
"obsolete id-right in msg-id"))
2049+
except errors.HeaderParseError:
2050+
raise errors.HeaderParseError(
2051+
"expected dot-atom-text, no-fold-literal or obs-id-right"
2052+
" but found '{}'".format(value))
2053+
msg_id.append(token)
2054+
if value and value[0] == '>':
2055+
value = value[1:]
2056+
else:
2057+
msg_id.defects.append(errors.InvalidHeaderDefect(
2058+
"missing trailing '>' on msg-id"))
2059+
msg_id.append(ValueTerminal('>', 'msg-id-end'))
2060+
if value and value[0] in CFWS_LEADER:
2061+
token, value = get_cfws(value)
2062+
msg_id.append(token)
2063+
return msg_id, value
2064+
2065+
2066+
def parse_message_id(value):
2067+
"""message-id = "Message-ID:" msg-id CRLF
2068+
"""
2069+
message_id = MessageID()
2070+
try:
2071+
token, value = get_msg_id(value)
2072+
except errors.HeaderParseError:
2073+
message_id.defects.append(errors.InvalidHeaderDefect(
2074+
"Expected msg-id but found {!r}".format(value)))
2075+
message_id.append(token)
2076+
return message_id
2077+
19712078
#
19722079
# XXX: As I begin to add additional header parsers, I'm realizing we probably
19732080
# have two level of parser routines: the get_XXX methods that get a token in

‎Lib/email/headerregistry.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,18 @@ def cte(self):
520520
return self._cte
521521

522522

523+
class MessageIDHeader:
524+
525+
max_count = 1
526+
value_parser = staticmethod(parser.parse_message_id)
527+
528+
@classmethod
529+
def parse(cls, value, kwds):
530+
kwds['parse_tree'] = parse_tree = cls.value_parser(value)
531+
kwds['decoded'] = str(parse_tree)
532+
kwds['defects'].extend(parse_tree.all_defects)
533+
534+
523535
# The header factory #
524536

525537
_default_header_map = {
@@ -542,6 +554,7 @@ def cte(self):
542554
'content-type': ContentTypeHeader,
543555
'content-disposition': ContentDispositionHeader,
544556
'content-transfer-encoding': ContentTransferEncodingHeader,
557+
'message-id': MessageIDHeader,
545558
}
546559

547560
class HeaderRegistry:

‎Lib/test/test_email/test__header_value_parser.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2494,6 +2494,78 @@ def test_invalid_content_transfer_encoding(self):
24942494
";foo", ";foo", ";foo", [errors.InvalidHeaderDefect]*3
24952495
)
24962496

2497+
# get_msg_id
2498+
2499+
def test_get_msg_id_valid(self):
2500+
msg_id = self._test_get_x(
2501+
parser.get_msg_id,
2502+
"<[email protected]>",
2503+
"<[email protected]>",
2504+
"<[email protected]>",
2505+
[],
2506+
'',
2507+
)
2508+
self.assertEqual(msg_id.token_type, 'msg-id')
2509+
2510+
def test_get_msg_id_obsolete_local(self):
2511+
msg_id = self._test_get_x(
2512+
parser.get_msg_id,
2513+
'<"simeple.local"@example.com>',
2514+
'<"simeple.local"@example.com>',
2515+
'<[email protected]>',
2516+
[errors.ObsoleteHeaderDefect],
2517+
'',
2518+
)
2519+
self.assertEqual(msg_id.token_type, 'msg-id')
2520+
2521+
def test_get_msg_id_non_folding_literal_domain(self):
2522+
msg_id = self._test_get_x(
2523+
parser.get_msg_id,
2524+
"<simple.local@[someexamplecom.domain]>",
2525+
"<simple.local@[someexamplecom.domain]>",
2526+
"<simple.local@[someexamplecom.domain]>",
2527+
[],
2528+
"",
2529+
)
2530+
self.assertEqual(msg_id.token_type, 'msg-id')
2531+
2532+
2533+
def test_get_msg_id_obsolete_domain_part(self):
2534+
msg_id = self._test_get_x(
2535+
parser.get_msg_id,
2536+
"<simplelocal@(old)example.com>",
2537+
"<simplelocal@(old)example.com>",
2538+
"<simplelocal@ example.com>",
2539+
[errors.ObsoleteHeaderDefect],
2540+
""
2541+
)
2542+
2543+
def test_get_msg_id_no_id_right_part(self):
2544+
msg_id = self._test_get_x(
2545+
parser.get_msg_id,
2546+
"<simplelocal>",
2547+
"<simplelocal>",
2548+
"<simplelocal>",
2549+
[errors.InvalidHeaderDefect],
2550+
""
2551+
)
2552+
self.assertEqual(msg_id.token_type, 'msg-id')
2553+
2554+
def test_get_msg_id_no_angle_start(self):
2555+
with self.assertRaises(errors.HeaderParseError):
2556+
parser.get_msg_id("msgwithnoankle")
2557+
2558+
def test_get_msg_id_no_angle_end(self):
2559+
msg_id = self._test_get_x(
2560+
parser.get_msg_id,
2561+
"<simplelocal@domain",
2562+
"<simplelocal@domain>",
2563+
"<simplelocal@domain>",
2564+
[errors.InvalidHeaderDefect],
2565+
""
2566+
)
2567+
self.assertEqual(msg_id.token_type, 'msg-id')
2568+
24972569

24982570
@parameterize
24992571
class Test_parse_mime_parameters(TestParserMixin, TestEmailBase):

0 commit comments

Comments
 (0)