Commit | Line | Data |
---|---|---|
920dae64 AT |
1 | # Copyright (C) 2001-2004 Python Software Foundation |
2 | # Author: Ben Gertzfield, Barry Warsaw | |
3 | # Contact: email-sig@python.org | |
4 | ||
5 | import email.base64MIME | |
6 | import email.quopriMIME | |
7 | from email.Encoders import encode_7or8bit | |
8 | ||
9 | ||
10 | \f | |
11 | # Flags for types of header encodings | |
12 | QP = 1 # Quoted-Printable | |
13 | BASE64 = 2 # Base64 | |
14 | SHORTEST = 3 # the shorter of QP and base64, but only for headers | |
15 | ||
16 | # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 | |
17 | MISC_LEN = 7 | |
18 | ||
19 | DEFAULT_CHARSET = 'us-ascii' | |
20 | ||
21 | ||
22 | \f | |
23 | # Defaults | |
24 | CHARSETS = { | |
25 | # input header enc body enc output conv | |
26 | 'iso-8859-1': (QP, QP, None), | |
27 | 'iso-8859-2': (QP, QP, None), | |
28 | 'iso-8859-3': (QP, QP, None), | |
29 | 'iso-8859-4': (QP, QP, None), | |
30 | # iso-8859-5 is Cyrillic, and not especially used | |
31 | # iso-8859-6 is Arabic, also not particularly used | |
32 | # iso-8859-7 is Greek, QP will not make it readable | |
33 | # iso-8859-8 is Hebrew, QP will not make it readable | |
34 | 'iso-8859-9': (QP, QP, None), | |
35 | 'iso-8859-10': (QP, QP, None), | |
36 | # iso-8859-11 is Thai, QP will not make it readable | |
37 | 'iso-8859-13': (QP, QP, None), | |
38 | 'iso-8859-14': (QP, QP, None), | |
39 | 'iso-8859-15': (QP, QP, None), | |
40 | 'windows-1252':(QP, QP, None), | |
41 | 'viscii': (QP, QP, None), | |
42 | 'us-ascii': (None, None, None), | |
43 | 'big5': (BASE64, BASE64, None), | |
44 | 'gb2312': (BASE64, BASE64, None), | |
45 | 'euc-jp': (BASE64, None, 'iso-2022-jp'), | |
46 | 'shift_jis': (BASE64, None, 'iso-2022-jp'), | |
47 | 'iso-2022-jp': (BASE64, None, None), | |
48 | 'koi8-r': (BASE64, BASE64, None), | |
49 | 'utf-8': (SHORTEST, BASE64, 'utf-8'), | |
50 | # We're making this one up to represent raw unencoded 8-bit | |
51 | '8bit': (None, BASE64, 'utf-8'), | |
52 | } | |
53 | ||
54 | # Aliases for other commonly-used names for character sets. Map | |
55 | # them to the real ones used in email. | |
56 | ALIASES = { | |
57 | 'latin_1': 'iso-8859-1', | |
58 | 'latin-1': 'iso-8859-1', | |
59 | 'latin_2': 'iso-8859-2', | |
60 | 'latin-2': 'iso-8859-2', | |
61 | 'latin_3': 'iso-8859-3', | |
62 | 'latin-3': 'iso-8859-3', | |
63 | 'latin_4': 'iso-8859-4', | |
64 | 'latin-4': 'iso-8859-4', | |
65 | 'latin_5': 'iso-8859-9', | |
66 | 'latin-5': 'iso-8859-9', | |
67 | 'latin_6': 'iso-8859-10', | |
68 | 'latin-6': 'iso-8859-10', | |
69 | 'latin_7': 'iso-8859-13', | |
70 | 'latin-7': 'iso-8859-13', | |
71 | 'latin_8': 'iso-8859-14', | |
72 | 'latin-8': 'iso-8859-14', | |
73 | 'latin_9': 'iso-8859-15', | |
74 | 'latin-9': 'iso-8859-15', | |
75 | 'cp949': 'ks_c_5601-1987', | |
76 | 'euc_jp': 'euc-jp', | |
77 | 'euc_kr': 'euc-kr', | |
78 | 'ascii': 'us-ascii', | |
79 | } | |
80 | ||
81 | ||
82 | # Map charsets to their Unicode codec strings. | |
83 | CODEC_MAP = { | |
84 | 'gb2312': 'eucgb2312_cn', | |
85 | 'big5': 'big5_tw', | |
86 | # Hack: We don't want *any* conversion for stuff marked us-ascii, as all | |
87 | # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. | |
88 | # Let that stuff pass through without conversion to/from Unicode. | |
89 | 'us-ascii': None, | |
90 | } | |
91 | ||
92 | ||
93 | \f | |
94 | # Convenience functions for extending the above mappings | |
95 | def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): | |
96 | """Add character set properties to the global registry. | |
97 | ||
98 | charset is the input character set, and must be the canonical name of a | |
99 | character set. | |
100 | ||
101 | Optional header_enc and body_enc is either Charset.QP for | |
102 | quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for | |
103 | the shortest of qp or base64 encoding, or None for no encoding. SHORTEST | |
104 | is only valid for header_enc. It describes how message headers and | |
105 | message bodies in the input charset are to be encoded. Default is no | |
106 | encoding. | |
107 | ||
108 | Optional output_charset is the character set that the output should be | |
109 | in. Conversions will proceed from input charset, to Unicode, to the | |
110 | output charset when the method Charset.convert() is called. The default | |
111 | is to output in the same character set as the input. | |
112 | ||
113 | Both input_charset and output_charset must have Unicode codec entries in | |
114 | the module's charset-to-codec mapping; use add_codec(charset, codecname) | |
115 | to add codecs the module does not know about. See the codecs module's | |
116 | documentation for more information. | |
117 | """ | |
118 | if body_enc == SHORTEST: | |
119 | raise ValueError('SHORTEST not allowed for body_enc') | |
120 | CHARSETS[charset] = (header_enc, body_enc, output_charset) | |
121 | ||
122 | ||
123 | def add_alias(alias, canonical): | |
124 | """Add a character set alias. | |
125 | ||
126 | alias is the alias name, e.g. latin-1 | |
127 | canonical is the character set's canonical name, e.g. iso-8859-1 | |
128 | """ | |
129 | ALIASES[alias] = canonical | |
130 | ||
131 | ||
132 | def add_codec(charset, codecname): | |
133 | """Add a codec that map characters in the given charset to/from Unicode. | |
134 | ||
135 | charset is the canonical name of a character set. codecname is the name | |
136 | of a Python codec, as appropriate for the second argument to the unicode() | |
137 | built-in, or to the encode() method of a Unicode string. | |
138 | """ | |
139 | CODEC_MAP[charset] = codecname | |
140 | ||
141 | ||
142 | \f | |
143 | class Charset: | |
144 | """Map character sets to their email properties. | |
145 | ||
146 | This class provides information about the requirements imposed on email | |
147 | for a specific character set. It also provides convenience routines for | |
148 | converting between character sets, given the availability of the | |
149 | applicable codecs. Given a character set, it will do its best to provide | |
150 | information on how to use that character set in an email in an | |
151 | RFC-compliant way. | |
152 | ||
153 | Certain character sets must be encoded with quoted-printable or base64 | |
154 | when used in email headers or bodies. Certain character sets must be | |
155 | converted outright, and are not allowed in email. Instances of this | |
156 | module expose the following information about a character set: | |
157 | ||
158 | input_charset: The initial character set specified. Common aliases | |
159 | are converted to their `official' email names (e.g. latin_1 | |
160 | is converted to iso-8859-1). Defaults to 7-bit us-ascii. | |
161 | ||
162 | header_encoding: If the character set must be encoded before it can be | |
163 | used in an email header, this attribute will be set to | |
164 | Charset.QP (for quoted-printable), Charset.BASE64 (for | |
165 | base64 encoding), or Charset.SHORTEST for the shortest of | |
166 | QP or BASE64 encoding. Otherwise, it will be None. | |
167 | ||
168 | body_encoding: Same as header_encoding, but describes the encoding for the | |
169 | mail message's body, which indeed may be different than the | |
170 | header encoding. Charset.SHORTEST is not allowed for | |
171 | body_encoding. | |
172 | ||
173 | output_charset: Some character sets must be converted before the can be | |
174 | used in email headers or bodies. If the input_charset is | |
175 | one of them, this attribute will contain the name of the | |
176 | charset output will be converted to. Otherwise, it will | |
177 | be None. | |
178 | ||
179 | input_codec: The name of the Python codec used to convert the | |
180 | input_charset to Unicode. If no conversion codec is | |
181 | necessary, this attribute will be None. | |
182 | ||
183 | output_codec: The name of the Python codec used to convert Unicode | |
184 | to the output_charset. If no conversion codec is necessary, | |
185 | this attribute will have the same value as the input_codec. | |
186 | """ | |
187 | def __init__(self, input_charset=DEFAULT_CHARSET): | |
188 | # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to | |
189 | # unicode because its .lower() is locale insensitive. | |
190 | input_charset = unicode(input_charset, 'ascii').lower() | |
191 | # Set the input charset after filtering through the aliases | |
192 | self.input_charset = ALIASES.get(input_charset, input_charset) | |
193 | # We can try to guess which encoding and conversion to use by the | |
194 | # charset_map dictionary. Try that first, but let the user override | |
195 | # it. | |
196 | henc, benc, conv = CHARSETS.get(self.input_charset, | |
197 | (SHORTEST, BASE64, None)) | |
198 | if not conv: | |
199 | conv = self.input_charset | |
200 | # Set the attributes, allowing the arguments to override the default. | |
201 | self.header_encoding = henc | |
202 | self.body_encoding = benc | |
203 | self.output_charset = ALIASES.get(conv, conv) | |
204 | # Now set the codecs. If one isn't defined for input_charset, | |
205 | # guess and try a Unicode codec with the same name as input_codec. | |
206 | self.input_codec = CODEC_MAP.get(self.input_charset, | |
207 | self.input_charset) | |
208 | self.output_codec = CODEC_MAP.get(self.output_charset, | |
209 | self.output_charset) | |
210 | ||
211 | def __str__(self): | |
212 | return self.input_charset.lower() | |
213 | ||
214 | __repr__ = __str__ | |
215 | ||
216 | def __eq__(self, other): | |
217 | return str(self) == str(other).lower() | |
218 | ||
219 | def __ne__(self, other): | |
220 | return not self.__eq__(other) | |
221 | ||
222 | def get_body_encoding(self): | |
223 | """Return the content-transfer-encoding used for body encoding. | |
224 | ||
225 | This is either the string `quoted-printable' or `base64' depending on | |
226 | the encoding used, or it is a function in which case you should call | |
227 | the function with a single argument, the Message object being | |
228 | encoded. The function should then set the Content-Transfer-Encoding | |
229 | header itself to whatever is appropriate. | |
230 | ||
231 | Returns "quoted-printable" if self.body_encoding is QP. | |
232 | Returns "base64" if self.body_encoding is BASE64. | |
233 | Returns "7bit" otherwise. | |
234 | """ | |
235 | assert self.body_encoding <> SHORTEST | |
236 | if self.body_encoding == QP: | |
237 | return 'quoted-printable' | |
238 | elif self.body_encoding == BASE64: | |
239 | return 'base64' | |
240 | else: | |
241 | return encode_7or8bit | |
242 | ||
243 | def convert(self, s): | |
244 | """Convert a string from the input_codec to the output_codec.""" | |
245 | if self.input_codec <> self.output_codec: | |
246 | return unicode(s, self.input_codec).encode(self.output_codec) | |
247 | else: | |
248 | return s | |
249 | ||
250 | def to_splittable(self, s): | |
251 | """Convert a possibly multibyte string to a safely splittable format. | |
252 | ||
253 | Uses the input_codec to try and convert the string to Unicode, so it | |
254 | can be safely split on character boundaries (even for multibyte | |
255 | characters). | |
256 | ||
257 | Returns the string as-is if it isn't known how to convert it to | |
258 | Unicode with the input_charset. | |
259 | ||
260 | Characters that could not be converted to Unicode will be replaced | |
261 | with the Unicode replacement character U+FFFD. | |
262 | """ | |
263 | if isinstance(s, unicode) or self.input_codec is None: | |
264 | return s | |
265 | try: | |
266 | return unicode(s, self.input_codec, 'replace') | |
267 | except LookupError: | |
268 | # Input codec not installed on system, so return the original | |
269 | # string unchanged. | |
270 | return s | |
271 | ||
272 | def from_splittable(self, ustr, to_output=True): | |
273 | """Convert a splittable string back into an encoded string. | |
274 | ||
275 | Uses the proper codec to try and convert the string from Unicode back | |
276 | into an encoded format. Return the string as-is if it is not Unicode, | |
277 | or if it could not be converted from Unicode. | |
278 | ||
279 | Characters that could not be converted from Unicode will be replaced | |
280 | with an appropriate character (usually '?'). | |
281 | ||
282 | If to_output is True (the default), uses output_codec to convert to an | |
283 | encoded format. If to_output is False, uses input_codec. | |
284 | """ | |
285 | if to_output: | |
286 | codec = self.output_codec | |
287 | else: | |
288 | codec = self.input_codec | |
289 | if not isinstance(ustr, unicode) or codec is None: | |
290 | return ustr | |
291 | try: | |
292 | return ustr.encode(codec, 'replace') | |
293 | except LookupError: | |
294 | # Output codec not installed | |
295 | return ustr | |
296 | ||
297 | def get_output_charset(self): | |
298 | """Return the output character set. | |
299 | ||
300 | This is self.output_charset if that is not None, otherwise it is | |
301 | self.input_charset. | |
302 | """ | |
303 | return self.output_charset or self.input_charset | |
304 | ||
305 | def encoded_header_len(self, s): | |
306 | """Return the length of the encoded header string.""" | |
307 | cset = self.get_output_charset() | |
308 | # The len(s) of a 7bit encoding is len(s) | |
309 | if self.header_encoding == BASE64: | |
310 | return email.base64MIME.base64_len(s) + len(cset) + MISC_LEN | |
311 | elif self.header_encoding == QP: | |
312 | return email.quopriMIME.header_quopri_len(s) + len(cset) + MISC_LEN | |
313 | elif self.header_encoding == SHORTEST: | |
314 | lenb64 = email.base64MIME.base64_len(s) | |
315 | lenqp = email.quopriMIME.header_quopri_len(s) | |
316 | return min(lenb64, lenqp) + len(cset) + MISC_LEN | |
317 | else: | |
318 | return len(s) | |
319 | ||
320 | def header_encode(self, s, convert=False): | |
321 | """Header-encode a string, optionally converting it to output_charset. | |
322 | ||
323 | If convert is True, the string will be converted from the input | |
324 | charset to the output charset automatically. This is not useful for | |
325 | multibyte character sets, which have line length issues (multibyte | |
326 | characters must be split on a character, not a byte boundary); use the | |
327 | high-level Header class to deal with these issues. convert defaults | |
328 | to False. | |
329 | ||
330 | The type of encoding (base64 or quoted-printable) will be based on | |
331 | self.header_encoding. | |
332 | """ | |
333 | cset = self.get_output_charset() | |
334 | if convert: | |
335 | s = self.convert(s) | |
336 | # 7bit/8bit encodings return the string unchanged (modulo conversions) | |
337 | if self.header_encoding == BASE64: | |
338 | return email.base64MIME.header_encode(s, cset) | |
339 | elif self.header_encoding == QP: | |
340 | return email.quopriMIME.header_encode(s, cset, maxlinelen=None) | |
341 | elif self.header_encoding == SHORTEST: | |
342 | lenb64 = email.base64MIME.base64_len(s) | |
343 | lenqp = email.quopriMIME.header_quopri_len(s) | |
344 | if lenb64 < lenqp: | |
345 | return email.base64MIME.header_encode(s, cset) | |
346 | else: | |
347 | return email.quopriMIME.header_encode(s, cset, maxlinelen=None) | |
348 | else: | |
349 | return s | |
350 | ||
351 | def body_encode(self, s, convert=True): | |
352 | """Body-encode a string and convert it to output_charset. | |
353 | ||
354 | If convert is True (the default), the string will be converted from | |
355 | the input charset to output charset automatically. Unlike | |
356 | header_encode(), there are no issues with byte boundaries and | |
357 | multibyte charsets in email bodies, so this is usually pretty safe. | |
358 | ||
359 | The type of encoding (base64 or quoted-printable) will be based on | |
360 | self.body_encoding. | |
361 | """ | |
362 | if convert: | |
363 | s = self.convert(s) | |
364 | # 7bit/8bit encodings return the string unchanged (module conversions) | |
365 | if self.body_encoding is BASE64: | |
366 | return email.base64MIME.body_encode(s) | |
367 | elif self.body_encoding is QP: | |
368 | return email.quopriMIME.body_encode(s) | |
369 | else: | |
370 | return s |