"""RFC 2822 message manipulation.
Note: This is only a very rough sketch of a full RFC-822 parser; in particular
the tokenizing of addresses does not adhere to all the quoting rules.
Note: RFC 2822 is a long awaited update to RFC 822. This module should
conform to RFC 2822, and is thus mis-named (it's not worth renaming it). Some
effort at RFC 2822 updates have been made, but a thorough audit has not been
performed. Consider any RFC 2822 non-conformance to be a bug.
RFC 2822: http://www.faqs.org/rfcs/rfc2822.html
RFC 822 : http://www.faqs.org/rfcs/rfc822.html (obsolete)
To create a Message object: first open a file, e.g.:
You can use any other legal way of getting an open file object, e.g. use
sys.stdin or call os.popen(). Then pass the open file object to the Message()
This class can work with any input object that supports a readline method. If
the input object has seek and tell capability, the rewindbody method will
work; also illegal lines will be pushed back onto the input stream. If the
input object lacks seek but has an `unread' method that can push back a line
of input, Message will use that to push back illegal lines. Thus this class
can be used to parse messages coming from a buffered stream.
The optional `seekable' argument is provided as a workaround for certain stdio
libraries in which tell() discards buffered data before discovering that the
lseek() system call doesn't work. For maximum portability, you should set the
seekable argument to zero to prevent that initial \code{tell} when passing in
an unseekable object such as a a file object created from a socket object. If
it is 1 on entry -- which it is by default -- the tell() method of the open
file object is called once; if this raises an exception, seekable is reset to
0. For other nonzero values of seekable, this test is not made.
To get the text of a particular header there are several methods:
str = m.getrawheader(name)
where name is the name of the header, e.g. 'Subject'. The difference is that
getheader() strips the leading and trailing whitespace, while getrawheader()
doesn't. Both functions retain embedded whitespace (including newlines)
exactly as they are specified in the header, and leave the case of the text
For addresses and address lists there are functions
realname, mailaddress = m.getaddr(name)
list = m.getaddrlist(name)
where the latter returns a list of (realname, mailaddr) tuples.
which parses a Date-like field and returns a time-compatible tuple,
i.e. a tuple such as returned by time.localtime() or accepted by
See the class definition for lower level access methods.
There are also some utility functions here.
# Cleanup and extensions by Eric S. Raymond <esr@thyrsus.com>
__all__
= ["Message","AddressList","parsedate","parsedate_tz","mktime_tz"]
_blanklines
= ('\r\n', '\n') # Optimization for islast()
"""Represents a single RFC 2822-compliant message."""
def __init__(self
, fp
, seekable
= 1):
"""Initialize the class instance and read the headers."""
# Exercise tell() to make sure it works
# (and then assume seek() works, too)
except (AttributeError, IOError):
self
.startofheaders
= None
self
.startofheaders
= self
.fp
.tell()
self
.startofbody
= self
.fp
.tell()
"""Rewind the file to the start of the body (if seekable)."""
raise IOError, "unseekable file"
self
.fp
.seek(self
.startofbody
)
Read header lines up to the entirely blank line that terminates them.
The (normally blank) line that ends the headers is skipped, but not
included in the returned list. If a non-header line ends the headers,
(which is an error), an attempt is made to backspace over it; it is
never included in the returned list.
The variable self.status is set to the empty string if all went well,
otherwise it is an error message. The variable self.headers is a
completely uninterpreted list of lines contained in the header (so
printing them will reproduce the header exactly as it appears in the
startofline
= unread
= tell
= None
if hasattr(self
.fp
, 'unread'):
startofline
= tell
= None
line
= self
.fp
.readline()
self
.status
= 'EOF in headers'
# Skip unix From name time lines
if firstline
and line
.startswith('From '):
self
.unixfrom
= self
.unixfrom
+ line
if headerseen
and line
[0] in ' \t':
# It's a continuation line.
x
= (self
.dict[headerseen
] + "\n " + line
.strip())
self
.dict[headerseen
] = x
.strip()
elif self
.iscomment(line
):
# It's a comment. Ignore it.
# Note! No pushback here! The delimiter line gets eaten.
headerseen
= self
.isheader(line
)
# It's a legal header line, save it.
self
.dict[headerseen
] = line
[len(headerseen
)+1:].strip()
# It's not a header line; throw it back and stop here.
self
.status
= 'No headers'
self
.status
= 'Non-header line where header expected'
self
.fp
.seek(startofline
)
self
.status
= self
.status
+ '; bad seek'
def isheader(self
, line
):
"""Determine whether a given line is a legal header.
This method should return the header name, suitably canonicalized.
You may override this method in order to use Message parsing on tagged
data in RFC 2822-like formats with special header formats.
"""Determine whether a line is a legal end of RFC 2822 headers.
You may override this method if your application wants to bend the
rules, e.g. to strip trailing whitespace, or to recognize MH template
separators ('--------'). For convenience (e.g. for code reading from
sockets) a line consisting of \r\n also matches.
return line
in _blanklines
def iscomment(self
, line
):
"""Determine whether a line should be skipped entirely.
You may override this method in order to use Message parsing on tagged
data in RFC 2822-like formats that support embedded comments or
def getallmatchingheaders(self
, name
):
"""Find all header lines matching a given header name.
Look through the list of headers and find all lines matching a given
header name (and their continuation lines). A list of the lines is
returned, without interpretation. If the header does not occur, an
empty list is returned. If the header occurs multiple times, all
occurrences are returned. Case is not important in the header name.
name
= name
.lower() + ':'
for line
in self
.headers
:
if line
[:n
].lower() == name
:
elif not line
[:1].isspace():
def getfirstmatchingheader(self
, name
):
"""Get the first header line matching name.
This is similar to getallmatchingheaders, but it returns only the
first matching header (and its continuation lines).
name
= name
.lower() + ':'
for line
in self
.headers
:
if not line
[:1].isspace():
elif line
[:n
].lower() == name
:
def getrawheader(self
, name
):
"""A higher-level interface to getfirstmatchingheader().
Return a string containing the literal text of the header but with the
keyword stripped. All leading, trailing and embedded whitespace is
kept in the string, however. Return None if the header does not
list = self
.getfirstmatchingheader(name
)
list[0] = list[0][len(name
) + 1:]
def getheader(self
, name
, default
=None):
"""Get the header value for a name.
This is the normal interface: it returns a stripped version of the
header value for a given header name, or None if it doesn't exist.
This uses the dictionary version which finds the *last* such header.
return self
.dict[name
.lower()]
def getheaders(self
, name
):
"""Get all values for a header.
This returns a list of values for headers given more than once; each
value in the result list is stripped in the same way as the result of
getheader(). If the header is not given, return an empty list.
for s
in self
.getallmatchingheaders(name
):
current
= "%s\n %s" % (current
, s
.strip())
current
= s
[s
.find(":") + 1:].strip()
"""Get a single address from a header, as a tuple.
('Guido van Rossum', 'guido@cwi.nl')
alist
= self
.getaddrlist(name
)
def getaddrlist(self
, name
):
"""Get a list of addresses from a header.
Retrieves a list of addresses from a header, where each address is a
tuple as returned by getaddr(). Scans all named headers, so it works
properly with multiple To: or Cc: headers for example.
for h
in self
.getallmatchingheaders(name
):
a
= AddressList(alladdrs
)
"""Retrieve a date field from a header.
Retrieves a date field from the named header, returning a tuple
compatible with time.mktime().
def getdate_tz(self
, name
):
"""Retrieve a date field from a header as a 10-tuple.
The first 9 elements make up a tuple compatible with time.mktime(),
and the 10th is the offset of the poster's time zone from GMT/UTC.
return parsedate_tz(data
)
# Access as a dictionary (only finds *last* header of each type):
"""Get the number of headers in a message."""
def __getitem__(self
, name
):
"""Get a specific header, as from a dictionary."""
return self
.dict[name
.lower()]
def __setitem__(self
, name
, value
):
"""Set the value of a header.
Note: This is not a perfect inversion of __getitem__, because any
changed headers get stuck at the end of the raw-headers list rather
than where the altered header was.
del self
[name
] # Won't fail if it doesn't exist
self
.dict[name
.lower()] = value
text
= name
+ ": " + value
self
.headers
.append(line
+ "\n")
def __delitem__(self
, name
):
"""Delete all occurrences of a specific header, if it is present."""
if not name
in self
.dict:
for i
in range(len(self
.headers
)):
if line
[:n
].lower() == name
:
elif not line
[:1].isspace():
def setdefault(self
, name
, default
=""):
if lowername
in self
.dict:
return self
.dict[lowername
]
text
= name
+ ": " + default
self
.headers
.append(line
+ "\n")
self
.dict[lowername
] = default
"""Determine whether a message contains the named header."""
return name
.lower() in self
.dict
def __contains__(self
, name
):
"""Determine whether a message contains the named header."""
return name
.lower() in self
.dict
"""Get all of a message's header field names."""
"""Get all of a message's header field values."""
return self
.dict.values()
"""Get all of a message's headers.
Returns a list of name, value tuples.
return ''.join(self
.headers
)
# XXX Should fix unquote() and quote() to be really conformant.
# XXX The inverses of the parse functions may also be useful.
"""Remove quotes from a string."""
if str.startswith('"') and str.endswith('"'):
return str[1:-1].replace('\\\\', '\\').replace('\\"', '"')
if str.startswith('<') and str.endswith('>'):
"""Add quotes around a string."""
return str.replace('\\', '\\\\').replace('"', '\\"')
"""Parse an address into a (realname, mailaddr) tuple."""
"""Address parser class by Ben Escoto.
To understand what this class does, it helps to have a copy of
RFC 2822 in front of you.
http://www.faqs.org/rfcs/rfc2822.html
Note: this class interface is deprecated and may be removed in the future.
Use rfc822.AddressList instead.
def __init__(self
, field
):
"""Initialize a new instance.
`field' is an unparsed address header field, containing one or more
self
.specials
= '()<>@,:;.\"[]'
self
.atomends
= self
.specials
+ self
.LWS
+ self
.CR
# Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it
# is obsolete syntax. RFC 2822 requires that we recognize obsolete
# syntax, so allow dots in phrases.
self
.phraseends
= self
.atomends
.replace('.', '')
"""Parse up to the start of the next address."""
while self
.pos
< len(self
.field
):
if self
.field
[self
.pos
] in self
.LWS
+ '\n\r':
elif self
.field
[self
.pos
] == '(':
self
.commentlist
.append(self
.getcomment())
Returns a list containing all of the addresses.
"""Parse the next address."""
plist
= self
.getphraselist()
if self
.pos
>= len(self
.field
):
# Bad email address technically, no domain.
returnlist
= [(' '.join(self
.commentlist
), plist
[0])]
elif self
.field
[self
.pos
] in '.@':
# email address is just an addrspec
# this isn't very efficient since we start over
addrspec
= self
.getaddrspec()
returnlist
= [(' '.join(self
.commentlist
), addrspec
)]
elif self
.field
[self
.pos
] == ':':
fieldlen
= len(self
.field
)
while self
.pos
< len(self
.field
):
if self
.pos
< fieldlen
and self
.field
[self
.pos
] == ';':
returnlist
= returnlist
+ self
.getaddress()
elif self
.field
[self
.pos
] == '<':
# Address is a phrase then a route addr
routeaddr
= self
.getrouteaddr()
returnlist
= [(' '.join(plist
) + ' (' + \
' '.join(self
.commentlist
) + ')', routeaddr
)]
else: returnlist
= [(' '.join(plist
), routeaddr
)]
returnlist
= [(' '.join(self
.commentlist
), plist
[0])]
elif self
.field
[self
.pos
] in self
.specials
:
if self
.pos
< len(self
.field
) and self
.field
[self
.pos
] == ',':
"""Parse a route address (Return-path value).
This method just skips all the route stuff and returns the addrspec.
if self
.field
[self
.pos
] != '<':
while self
.pos
< len(self
.field
):
elif self
.field
[self
.pos
] == '>':
elif self
.field
[self
.pos
] == '@':
elif self
.field
[self
.pos
] == ':':
adlist
= self
.getaddrspec()
"""Parse an RFC 2822 addr-spec."""
while self
.pos
< len(self
.field
):
if self
.field
[self
.pos
] == '.':
elif self
.field
[self
.pos
] == '"':
aslist
.append('"%s"' % self
.getquote())
elif self
.field
[self
.pos
] in self
.atomends
:
else: aslist
.append(self
.getatom())
if self
.pos
>= len(self
.field
) or self
.field
[self
.pos
] != '@':
return ''.join(aslist
) + self
.getdomain()
"""Get the complete domain name from an address."""
while self
.pos
< len(self
.field
):
if self
.field
[self
.pos
] in self
.LWS
:
elif self
.field
[self
.pos
] == '(':
self
.commentlist
.append(self
.getcomment())
elif self
.field
[self
.pos
] == '[':
sdlist
.append(self
.getdomainliteral())
elif self
.field
[self
.pos
] == '.':
elif self
.field
[self
.pos
] in self
.atomends
:
else: sdlist
.append(self
.getatom())
def getdelimited(self
, beginchar
, endchars
, allowcomments
= 1):
"""Parse a header fragment delimited by special characters.
`beginchar' is the start character for the fragment. If self is not
looking at an instance of `beginchar' then getdelimited returns the
`endchars' is a sequence of allowable end-delimiting characters.
Parsing stops when one of these is encountered.
If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed
within the parsed fragment.
if self
.field
[self
.pos
] != beginchar
:
while self
.pos
< len(self
.field
):
slist
.append(self
.field
[self
.pos
])
elif self
.field
[self
.pos
] in endchars
:
elif allowcomments
and self
.field
[self
.pos
] == '(':
slist
.append(self
.getcomment())
elif self
.field
[self
.pos
] == '\\':
slist
.append(self
.field
[self
.pos
])
"""Get a quote-delimited fragment from self's field."""
return self
.getdelimited('"', '"\r', 0)
"""Get a parenthesis-delimited fragment from self's field."""
return self
.getdelimited('(', ')\r', 1)
def getdomainliteral(self
):
"""Parse an RFC 2822 domain-literal."""
return '[%s]' % self
.getdelimited('[', ']\r', 0)
def getatom(self
, atomends
=None):
"""Parse an RFC 2822 atom.
Optional atomends specifies a different set of end token delimiters
(the default is to use self.atomends). This is used e.g. in
getphraselist() since phrase endings must not include the `.' (which
while self
.pos
< len(self
.field
):
if self
.field
[self
.pos
] in atomends
:
else: atomlist
.append(self
.field
[self
.pos
])
"""Parse a sequence of RFC 2822 phrases.
A phrase is a sequence of words, which are in turn either RFC 2822
atoms or quoted-strings. Phrases are canonicalized by squeezing all
runs of continuous whitespace into one space.
while self
.pos
< len(self
.field
):
if self
.field
[self
.pos
] in self
.LWS
:
elif self
.field
[self
.pos
] == '"':
plist
.append(self
.getquote())
elif self
.field
[self
.pos
] == '(':
self
.commentlist
.append(self
.getcomment())
elif self
.field
[self
.pos
] in self
.phraseends
:
plist
.append(self
.getatom(self
.phraseends
))
class AddressList(AddrlistClass
):
"""An AddressList encapsulates a list of parsed RFC 2822 addresses."""
def __init__(self
, field
):
AddrlistClass
.__init
__(self
, field
)
self
.addresslist
= self
.getaddrlist()
return len(self
.addresslist
)
return ", ".join(map(dump_address_pair
, self
.addresslist
))
def __add__(self
, other
):
newaddr
= AddressList(None)
newaddr
.addresslist
= self
.addresslist
[:]
for x
in other
.addresslist
:
if not x
in self
.addresslist
:
newaddr
.addresslist
.append(x
)
def __iadd__(self
, other
):
for x
in other
.addresslist
:
if not x
in self
.addresslist
:
self
.addresslist
.append(x
)
def __sub__(self
, other
):
newaddr
= AddressList(None)
for x
in self
.addresslist
:
if not x
in other
.addresslist
:
newaddr
.addresslist
.append(x
)
def __isub__(self
, other
):
# Set difference, in-place
for x
in other
.addresslist
:
if x
in self
.addresslist
:
self
.addresslist
.remove(x
)
def __getitem__(self
, index
):
# Make indexing, slices, and 'in' work
return self
.addresslist
[index
]
def dump_address_pair(pair
):
"""Dump a (name, address) pair in a canonicalized form."""
return '"' + pair
[0] + '" <' + pair
[1] + '>'
_monthnames
= ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul',
'aug', 'sep', 'oct', 'nov', 'dec',
'january', 'february', 'march', 'april', 'may', 'june', 'july',
'august', 'september', 'october', 'november', 'december']
_daynames
= ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
# The timezone table does not include the military time zones defined
# in RFC822, other than Z. According to RFC1123, the description in
# RFC822 gets the signs wrong, so we can't rely on any such time
# zones. RFC1123 recommends that numeric timezone indicators be used
# instead of timezone names.
_timezones
= {'UT':0, 'UTC':0, 'GMT':0, 'Z':0,
'AST': -400, 'ADT': -300, # Atlantic (used in Canada)
'EST': -500, 'EDT': -400, # Eastern
'CST': -600, 'CDT': -500, # Central
'MST': -700, 'MDT': -600, # Mountain
'PST': -800, 'PDT': -700 # Pacific
"""Convert a date string to a time tuple.
Accounts for military timezones.
if data
[0][-1] in (',', '.') or data
[0].lower() in _daynames
:
# There's a dayname here. Skip it
if len(data
) == 3: # RFC 850 date, deprecated
stuff
= data
[0].split('-')
data
[3:] = [s
[:i
], s
[i
+1:]]
data
.append('') # Dummy tz
[dd
, mm
, yy
, tm
, tz
] = data
if not mm
in _monthnames
:
if not mm
in _monthnames
:
mm
= _monthnames
.index(mm
)+1
tzoffset
= _timezones
[tz
]
# Convert a timezone offset into seconds ; -0500 -> -18000
tzoffset
= tzsign
* ( (tzoffset
//100)*3600 + (tzoffset
% 100)*60)
tuple = (yy
, mm
, dd
, thh
, tmm
, tss
, 0, 1, 0, tzoffset
)
"""Convert a time string to a time tuple."""
if type(t
) == type( () ):
"""Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp."""
# No zone info, so localtime is better assumption than GMT
return time
.mktime(data
[:8] + (-1,))
t
= time
.mktime(data
[:8] + (0,))
return t
- data
[9] - time
.timezone
def formatdate(timeval
=None):
"""Returns time format preferred for Internet standards.
Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123
According to RFC 1123, day and month names must always be in
English. If not for that, this code could use strftime(). It
can't because strftime() honors the locale and could generated
timeval
= time
.gmtime(timeval
)
return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][timeval
[6]],
["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][timeval
[1]-1],
timeval
[0], timeval
[3], timeval
[4], timeval
[5])
# When used as script, run a small test program.
# The first command line argument must be a filename containing one
# message in RFC-822 format.
if __name__
== '__main__':
file = os
.path
.join(os
.environ
['HOME'], 'Mail/inbox/1')
if sys
.argv
[1:]: file = sys
.argv
[1]
print 'From:', m
.getaddr('from')
print 'To:', m
.getaddrlist('to')
print 'Subject:', m
.getheader('subject')
print 'Date:', m
.getheader('date')
date
= m
.getdate_tz('date')
date
= time
.localtime(mktime_tz(date
))
print 'ParsedDate:', time
.asctime(date
),
hhmm
, ss
= divmod(hhmmss
, 60)
hh
, mm
= divmod(hhmm
, 60)
print "%+03d%02d" % (hh
, mm
),
if ss
: print ".%02d" % ss
,
print 'ParsedDate:', None
if 'Date' in m
: print 'Date =', m
['Date']
if 'X-Nonsense' in m
: pass
print 'values =', m
.values()
print 'items =', m
.items()