Commit | Line | Data |
---|---|---|
920dae64 AT |
1 | from test import test_support |
2 | import unittest | |
3 | import codecs | |
4 | import sys, StringIO | |
5 | ||
6 | class Queue(object): | |
7 | """ | |
8 | queue: write bytes at one end, read bytes from the other end | |
9 | """ | |
10 | def __init__(self): | |
11 | self._buffer = "" | |
12 | ||
13 | def write(self, chars): | |
14 | self._buffer += chars | |
15 | ||
16 | def read(self, size=-1): | |
17 | if size<0: | |
18 | s = self._buffer | |
19 | self._buffer = "" | |
20 | return s | |
21 | else: | |
22 | s = self._buffer[:size] | |
23 | self._buffer = self._buffer[size:] | |
24 | return s | |
25 | ||
26 | class ReadTest(unittest.TestCase): | |
27 | def test_seek(self): | |
28 | # all codecs should be able to encode these | |
29 | s = u"%s\n%s\n" % (100*u"abc123", 100*u"def456") | |
30 | encoding = self.encoding | |
31 | reader = codecs.getreader(encoding)(StringIO.StringIO(s.encode(encoding))) | |
32 | for t in xrange(5): | |
33 | # Test that calling seek resets the internal codec state and buffers | |
34 | reader.seek(0, 0) | |
35 | line = reader.readline() | |
36 | self.assertEqual(s[:len(line)], line) | |
37 | ||
38 | def check_partial(self, input, partialresults): | |
39 | # get a StreamReader for the encoding and feed the bytestring version | |
40 | # of input to the reader byte by byte. Read every available from | |
41 | # the StreamReader and check that the results equal the appropriate | |
42 | # entries from partialresults. | |
43 | q = Queue() | |
44 | r = codecs.getreader(self.encoding)(q) | |
45 | result = u"" | |
46 | for (c, partialresult) in zip(input.encode(self.encoding), partialresults): | |
47 | q.write(c) | |
48 | result += r.read() | |
49 | self.assertEqual(result, partialresult) | |
50 | # check that there's nothing left in the buffers | |
51 | self.assertEqual(r.read(), u"") | |
52 | self.assertEqual(r.bytebuffer, "") | |
53 | self.assertEqual(r.charbuffer, u"") | |
54 | ||
55 | def test_readline(self): | |
56 | def getreader(input): | |
57 | stream = StringIO.StringIO(input.encode(self.encoding)) | |
58 | return codecs.getreader(self.encoding)(stream) | |
59 | ||
60 | def readalllines(input, keepends=True): | |
61 | reader = getreader(input) | |
62 | lines = [] | |
63 | while True: | |
64 | line = reader.readline(keepends=keepends) | |
65 | if not line: | |
66 | break | |
67 | lines.append(line) | |
68 | return "".join(lines) | |
69 | ||
70 | s = u"foo\nbar\r\nbaz\rspam\u2028eggs" | |
71 | self.assertEqual(readalllines(s, True), s) | |
72 | self.assertEqual(readalllines(s, False), u"foobarbazspameggs") | |
73 | ||
74 | # Test long lines (multiple calls to read() in readline()) | |
75 | vw = [] | |
76 | vwo = [] | |
77 | for (i, lineend) in enumerate(u"\n \r\n \r \u2028".split()): | |
78 | vw.append((i*200)*u"\3042" + lineend) | |
79 | vwo.append((i*200)*u"\3042") | |
80 | self.assertEqual(readalllines("".join(vw), True), "".join(vw)) | |
81 | self.assertEqual(readalllines("".join(vw), False),"".join(vwo)) | |
82 | ||
83 | # Test lines where the first read might end with \r, so the | |
84 | # reader has to look ahead whether this is a lone \r or a \r\n | |
85 | for size in xrange(80): | |
86 | for lineend in u"\n \r\n \r \u2028".split(): | |
87 | s = 10*(size*u"a" + lineend + u"xxx\n") | |
88 | reader = getreader(s) | |
89 | for i in xrange(10): | |
90 | self.assertEqual( | |
91 | reader.readline(keepends=True), | |
92 | size*u"a" + lineend, | |
93 | ) | |
94 | reader = getreader(s) | |
95 | for i in xrange(10): | |
96 | self.assertEqual( | |
97 | reader.readline(keepends=False), | |
98 | size*u"a", | |
99 | ) | |
100 | ||
101 | def test_bug1175396(self): | |
102 | s = [ | |
103 | '<%!--===================================================\r\n', | |
104 | ' BLOG index page: show recent articles,\r\n', | |
105 | ' today\'s articles, or articles of a specific date.\r\n', | |
106 | '========================================================--%>\r\n', | |
107 | '<%@inputencoding="ISO-8859-1"%>\r\n', | |
108 | '<%@pagetemplate=TEMPLATE.y%>\r\n', | |
109 | '<%@import=import frog.util, frog%>\r\n', | |
110 | '<%@import=import frog.objects%>\r\n', | |
111 | '<%@import=from frog.storageerrors import StorageError%>\r\n', | |
112 | '<%\r\n', | |
113 | '\r\n', | |
114 | 'import logging\r\n', | |
115 | 'log=logging.getLogger("Snakelets.logger")\r\n', | |
116 | '\r\n', | |
117 | '\r\n', | |
118 | 'user=self.SessionCtx.user\r\n', | |
119 | 'storageEngine=self.SessionCtx.storageEngine\r\n', | |
120 | '\r\n', | |
121 | '\r\n', | |
122 | 'def readArticlesFromDate(date, count=None):\r\n', | |
123 | ' entryids=storageEngine.listBlogEntries(date)\r\n', | |
124 | ' entryids.reverse() # descending\r\n', | |
125 | ' if count:\r\n', | |
126 | ' entryids=entryids[:count]\r\n', | |
127 | ' try:\r\n', | |
128 | ' return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n', | |
129 | ' except StorageError,x:\r\n', | |
130 | ' log.error("Error loading articles: "+str(x))\r\n', | |
131 | ' self.abort("cannot load articles")\r\n', | |
132 | '\r\n', | |
133 | 'showdate=None\r\n', | |
134 | '\r\n', | |
135 | 'arg=self.Request.getArg()\r\n', | |
136 | 'if arg=="today":\r\n', | |
137 | ' #-------------------- TODAY\'S ARTICLES\r\n', | |
138 | ' self.write("<h2>Today\'s articles</h2>")\r\n', | |
139 | ' showdate = frog.util.isodatestr() \r\n', | |
140 | ' entries = readArticlesFromDate(showdate)\r\n', | |
141 | 'elif arg=="active":\r\n', | |
142 | ' #-------------------- ACTIVE ARTICLES redirect\r\n', | |
143 | ' self.Yredirect("active.y")\r\n', | |
144 | 'elif arg=="login":\r\n', | |
145 | ' #-------------------- LOGIN PAGE redirect\r\n', | |
146 | ' self.Yredirect("login.y")\r\n', | |
147 | 'elif arg=="date":\r\n', | |
148 | ' #-------------------- ARTICLES OF A SPECIFIC DATE\r\n', | |
149 | ' showdate = self.Request.getParameter("date")\r\n', | |
150 | ' self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n', | |
151 | ' entries = readArticlesFromDate(showdate)\r\n', | |
152 | 'else:\r\n', | |
153 | ' #-------------------- RECENT ARTICLES\r\n', | |
154 | ' self.write("<h2>Recent articles</h2>")\r\n', | |
155 | ' dates=storageEngine.listBlogEntryDates()\r\n', | |
156 | ' if dates:\r\n', | |
157 | ' entries=[]\r\n', | |
158 | ' SHOWAMOUNT=10\r\n', | |
159 | ' for showdate in dates:\r\n', | |
160 | ' entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n', | |
161 | ' if len(entries)>=SHOWAMOUNT:\r\n', | |
162 | ' break\r\n', | |
163 | ' \r\n', | |
164 | ] | |
165 | stream = StringIO.StringIO("".join(s).encode(self.encoding)) | |
166 | reader = codecs.getreader(self.encoding)(stream) | |
167 | for (i, line) in enumerate(reader): | |
168 | self.assertEqual(line, s[i]) | |
169 | ||
170 | def test_readlinequeue(self): | |
171 | q = Queue() | |
172 | writer = codecs.getwriter(self.encoding)(q) | |
173 | reader = codecs.getreader(self.encoding)(q) | |
174 | ||
175 | # No lineends | |
176 | writer.write(u"foo\r") | |
177 | self.assertEqual(reader.readline(keepends=False), u"foo") | |
178 | writer.write(u"\nbar\r") | |
179 | self.assertEqual(reader.readline(keepends=False), u"") | |
180 | self.assertEqual(reader.readline(keepends=False), u"bar") | |
181 | writer.write(u"baz") | |
182 | self.assertEqual(reader.readline(keepends=False), u"baz") | |
183 | self.assertEqual(reader.readline(keepends=False), u"") | |
184 | ||
185 | # Lineends | |
186 | writer.write(u"foo\r") | |
187 | self.assertEqual(reader.readline(keepends=True), u"foo\r") | |
188 | writer.write(u"\nbar\r") | |
189 | self.assertEqual(reader.readline(keepends=True), u"\n") | |
190 | self.assertEqual(reader.readline(keepends=True), u"bar\r") | |
191 | writer.write(u"baz") | |
192 | self.assertEqual(reader.readline(keepends=True), u"baz") | |
193 | self.assertEqual(reader.readline(keepends=True), u"") | |
194 | writer.write(u"foo\r\n") | |
195 | self.assertEqual(reader.readline(keepends=True), u"foo\r\n") | |
196 | ||
197 | def test_bug1098990_a(self): | |
198 | s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n" | |
199 | s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n" | |
200 | s3 = u"next line.\r\n" | |
201 | ||
202 | s = (s1+s2+s3).encode(self.encoding) | |
203 | stream = StringIO.StringIO(s) | |
204 | reader = codecs.getreader(self.encoding)(stream) | |
205 | self.assertEqual(reader.readline(), s1) | |
206 | self.assertEqual(reader.readline(), s2) | |
207 | self.assertEqual(reader.readline(), s3) | |
208 | self.assertEqual(reader.readline(), u"") | |
209 | ||
210 | def test_bug1098990_b(self): | |
211 | s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n" | |
212 | s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n" | |
213 | s3 = u"stillokay:bbbbxx\r\n" | |
214 | s4 = u"broken!!!!badbad\r\n" | |
215 | s5 = u"againokay.\r\n" | |
216 | ||
217 | s = (s1+s2+s3+s4+s5).encode(self.encoding) | |
218 | stream = StringIO.StringIO(s) | |
219 | reader = codecs.getreader(self.encoding)(stream) | |
220 | self.assertEqual(reader.readline(), s1) | |
221 | self.assertEqual(reader.readline(), s2) | |
222 | self.assertEqual(reader.readline(), s3) | |
223 | self.assertEqual(reader.readline(), s4) | |
224 | self.assertEqual(reader.readline(), s5) | |
225 | self.assertEqual(reader.readline(), u"") | |
226 | ||
227 | class UTF16Test(ReadTest): | |
228 | encoding = "utf-16" | |
229 | ||
230 | spamle = '\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00' | |
231 | spambe = '\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m' | |
232 | ||
233 | def test_only_one_bom(self): | |
234 | _,_,reader,writer = codecs.lookup(self.encoding) | |
235 | # encode some stream | |
236 | s = StringIO.StringIO() | |
237 | f = writer(s) | |
238 | f.write(u"spam") | |
239 | f.write(u"spam") | |
240 | d = s.getvalue() | |
241 | # check whether there is exactly one BOM in it | |
242 | self.assert_(d == self.spamle or d == self.spambe) | |
243 | # try to read it back | |
244 | s = StringIO.StringIO(d) | |
245 | f = reader(s) | |
246 | self.assertEquals(f.read(), u"spamspam") | |
247 | ||
248 | def test_partial(self): | |
249 | self.check_partial( | |
250 | u"\x00\xff\u0100\uffff", | |
251 | [ | |
252 | u"", # first byte of BOM read | |
253 | u"", # second byte of BOM read => byteorder known | |
254 | u"", | |
255 | u"\x00", | |
256 | u"\x00", | |
257 | u"\x00\xff", | |
258 | u"\x00\xff", | |
259 | u"\x00\xff\u0100", | |
260 | u"\x00\xff\u0100", | |
261 | u"\x00\xff\u0100\uffff", | |
262 | ] | |
263 | ) | |
264 | ||
265 | class UTF16LETest(ReadTest): | |
266 | encoding = "utf-16-le" | |
267 | ||
268 | def test_partial(self): | |
269 | self.check_partial( | |
270 | u"\x00\xff\u0100\uffff", | |
271 | [ | |
272 | u"", | |
273 | u"\x00", | |
274 | u"\x00", | |
275 | u"\x00\xff", | |
276 | u"\x00\xff", | |
277 | u"\x00\xff\u0100", | |
278 | u"\x00\xff\u0100", | |
279 | u"\x00\xff\u0100\uffff", | |
280 | ] | |
281 | ) | |
282 | ||
283 | class UTF16BETest(ReadTest): | |
284 | encoding = "utf-16-be" | |
285 | ||
286 | def test_partial(self): | |
287 | self.check_partial( | |
288 | u"\x00\xff\u0100\uffff", | |
289 | [ | |
290 | u"", | |
291 | u"\x00", | |
292 | u"\x00", | |
293 | u"\x00\xff", | |
294 | u"\x00\xff", | |
295 | u"\x00\xff\u0100", | |
296 | u"\x00\xff\u0100", | |
297 | u"\x00\xff\u0100\uffff", | |
298 | ] | |
299 | ) | |
300 | ||
301 | class UTF8Test(ReadTest): | |
302 | encoding = "utf-8" | |
303 | ||
304 | def test_partial(self): | |
305 | self.check_partial( | |
306 | u"\x00\xff\u07ff\u0800\uffff", | |
307 | [ | |
308 | u"\x00", | |
309 | u"\x00", | |
310 | u"\x00\xff", | |
311 | u"\x00\xff", | |
312 | u"\x00\xff\u07ff", | |
313 | u"\x00\xff\u07ff", | |
314 | u"\x00\xff\u07ff", | |
315 | u"\x00\xff\u07ff\u0800", | |
316 | u"\x00\xff\u07ff\u0800", | |
317 | u"\x00\xff\u07ff\u0800", | |
318 | u"\x00\xff\u07ff\u0800\uffff", | |
319 | ] | |
320 | ) | |
321 | ||
322 | class EscapeDecodeTest(unittest.TestCase): | |
323 | def test_empty_escape_decode(self): | |
324 | self.assertEquals(codecs.escape_decode(""), ("", 0)) | |
325 | ||
326 | class RecodingTest(unittest.TestCase): | |
327 | def test_recoding(self): | |
328 | f = StringIO.StringIO() | |
329 | f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8") | |
330 | f2.write(u"a") | |
331 | f2.close() | |
332 | # Python used to crash on this at exit because of a refcount | |
333 | # bug in _codecsmodule.c | |
334 | ||
335 | # From RFC 3492 | |
336 | punycode_testcases = [ | |
337 | # A Arabic (Egyptian): | |
338 | (u"\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644" | |
339 | u"\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F", | |
340 | "egbpdaj6bu4bxfgehfvwxn"), | |
341 | # B Chinese (simplified): | |
342 | (u"\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587", | |
343 | "ihqwcrb4cv8a8dqg056pqjye"), | |
344 | # C Chinese (traditional): | |
345 | (u"\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587", | |
346 | "ihqwctvzc91f659drss3x8bo0yb"), | |
347 | # D Czech: Pro<ccaron>prost<ecaron>nemluv<iacute><ccaron>esky | |
348 | (u"\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074" | |
349 | u"\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D" | |
350 | u"\u0065\u0073\u006B\u0079", | |
351 | "Proprostnemluvesky-uyb24dma41a"), | |
352 | # E Hebrew: | |
353 | (u"\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8" | |
354 | u"\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2" | |
355 | u"\u05D1\u05E8\u05D9\u05EA", | |
356 | "4dbcagdahymbxekheh6e0a7fei0b"), | |
357 | # F Hindi (Devanagari): | |
358 | (u"\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D" | |
359 | u"\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939" | |
360 | u"\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947" | |
361 | u"\u0939\u0948\u0902", | |
362 | "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"), | |
363 | ||
364 | #(G) Japanese (kanji and hiragana): | |
365 | (u"\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092" | |
366 | u"\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B", | |
367 | "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"), | |
368 | ||
369 | # (H) Korean (Hangul syllables): | |
370 | (u"\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774" | |
371 | u"\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74" | |
372 | u"\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C", | |
373 | "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j" | |
374 | "psd879ccm6fea98c"), | |
375 | ||
376 | # (I) Russian (Cyrillic): | |
377 | (u"\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E" | |
378 | u"\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440" | |
379 | u"\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A" | |
380 | u"\u0438", | |
381 | "b1abfaaepdrnnbgefbaDotcwatmq2g4l"), | |
382 | ||
383 | # (J) Spanish: Porqu<eacute>nopuedensimplementehablarenEspa<ntilde>ol | |
384 | (u"\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070" | |
385 | u"\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070" | |
386 | u"\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061" | |
387 | u"\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070" | |
388 | u"\u0061\u00F1\u006F\u006C", | |
389 | "PorqunopuedensimplementehablarenEspaol-fmd56a"), | |
390 | ||
391 | # (K) Vietnamese: | |
392 | # T<adotbelow>isaoh<odotbelow>kh<ocirc>ngth<ecirchookabove>ch\ | |
393 | # <ihookabove>n<oacute>iti<ecircacute>ngVi<ecircdotbelow>t | |
394 | (u"\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B" | |
395 | u"\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068" | |
396 | u"\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067" | |
397 | u"\u0056\u0069\u1EC7\u0074", | |
398 | "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"), | |
399 | ||
400 | ||
401 | #(L) 3<nen>B<gumi><kinpachi><sensei> | |
402 | (u"\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F", | |
403 | "3B-ww4c5e180e575a65lsy2b"), | |
404 | ||
405 | # (M) <amuro><namie>-with-SUPER-MONKEYS | |
406 | (u"\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074" | |
407 | u"\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D" | |
408 | u"\u004F\u004E\u004B\u0045\u0059\u0053", | |
409 | "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"), | |
410 | ||
411 | # (N) Hello-Another-Way-<sorezore><no><basho> | |
412 | (u"\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F" | |
413 | u"\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D" | |
414 | u"\u305D\u308C\u305E\u308C\u306E\u5834\u6240", | |
415 | "Hello-Another-Way--fc4qua05auwb3674vfr0b"), | |
416 | ||
417 | # (O) <hitotsu><yane><no><shita>2 | |
418 | (u"\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032", | |
419 | "2-u9tlzr9756bt3uc0v"), | |
420 | ||
421 | # (P) Maji<de>Koi<suru>5<byou><mae> | |
422 | (u"\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059" | |
423 | u"\u308B\u0035\u79D2\u524D", | |
424 | "MajiKoi5-783gue6qz075azm5e"), | |
425 | ||
426 | # (Q) <pafii>de<runba> | |
427 | (u"\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0", | |
428 | "de-jg4avhby1noc0d"), | |
429 | ||
430 | # (R) <sono><supiido><de> | |
431 | (u"\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067", | |
432 | "d9juau41awczczp"), | |
433 | ||
434 | # (S) -> $1.00 <- | |
435 | (u"\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020" | |
436 | u"\u003C\u002D", | |
437 | "-> $1.00 <--") | |
438 | ] | |
439 | ||
440 | for i in punycode_testcases: | |
441 | if len(i)!=2: | |
442 | print repr(i) | |
443 | ||
444 | class PunycodeTest(unittest.TestCase): | |
445 | def test_encode(self): | |
446 | for uni, puny in punycode_testcases: | |
447 | # Need to convert both strings to lower case, since | |
448 | # some of the extended encodings use upper case, but our | |
449 | # code produces only lower case. Converting just puny to | |
450 | # lower is also insufficient, since some of the input characters | |
451 | # are upper case. | |
452 | self.assertEquals(uni.encode("punycode").lower(), puny.lower()) | |
453 | ||
454 | def test_decode(self): | |
455 | for uni, puny in punycode_testcases: | |
456 | self.assertEquals(uni, puny.decode("punycode")) | |
457 | ||
458 | class UnicodeInternalTest(unittest.TestCase): | |
459 | def test_bug1251300(self): | |
460 | # Decoding with unicode_internal used to not correctly handle "code | |
461 | # points" above 0x10ffff on UCS-4 builds. | |
462 | if sys.maxunicode > 0xffff: | |
463 | ok = [ | |
464 | ("\x00\x10\xff\xff", u"\U0010ffff"), | |
465 | ("\x00\x00\x01\x01", u"\U00000101"), | |
466 | ("", u""), | |
467 | ] | |
468 | not_ok = [ | |
469 | "\x7f\xff\xff\xff", | |
470 | "\x80\x00\x00\x00", | |
471 | "\x81\x00\x00\x00", | |
472 | "\x00", | |
473 | "\x00\x00\x00\x00\x00", | |
474 | ] | |
475 | for internal, uni in ok: | |
476 | if sys.byteorder == "little": | |
477 | internal = "".join(reversed(internal)) | |
478 | self.assertEquals(uni, internal.decode("unicode_internal")) | |
479 | for internal in not_ok: | |
480 | if sys.byteorder == "little": | |
481 | internal = "".join(reversed(internal)) | |
482 | self.assertRaises(UnicodeDecodeError, internal.decode, | |
483 | "unicode_internal") | |
484 | ||
485 | def test_decode_error_attributes(self): | |
486 | if sys.maxunicode > 0xffff: | |
487 | try: | |
488 | "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") | |
489 | except UnicodeDecodeError, ex: | |
490 | self.assertEquals("unicode_internal", ex.encoding) | |
491 | self.assertEquals("\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) | |
492 | self.assertEquals(4, ex.start) | |
493 | self.assertEquals(8, ex.end) | |
494 | else: | |
495 | self.fail() | |
496 | ||
497 | def test_decode_callback(self): | |
498 | if sys.maxunicode > 0xffff: | |
499 | codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) | |
500 | decoder = codecs.getdecoder("unicode_internal") | |
501 | ab = u"ab".encode("unicode_internal") | |
502 | ignored = decoder("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), | |
503 | "UnicodeInternalTest") | |
504 | self.assertEquals((u"ab", 12), ignored) | |
505 | ||
506 | # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html | |
507 | nameprep_tests = [ | |
508 | # 3.1 Map to nothing. | |
509 | ('foo\xc2\xad\xcd\x8f\xe1\xa0\x86\xe1\xa0\x8bbar' | |
510 | '\xe2\x80\x8b\xe2\x81\xa0baz\xef\xb8\x80\xef\xb8\x88\xef' | |
511 | '\xb8\x8f\xef\xbb\xbf', | |
512 | 'foobarbaz'), | |
513 | # 3.2 Case folding ASCII U+0043 U+0041 U+0046 U+0045. | |
514 | ('CAFE', | |
515 | 'cafe'), | |
516 | # 3.3 Case folding 8bit U+00DF (german sharp s). | |
517 | # The original test case is bogus; it says \xc3\xdf | |
518 | ('\xc3\x9f', | |
519 | 'ss'), | |
520 | # 3.4 Case folding U+0130 (turkish capital I with dot). | |
521 | ('\xc4\xb0', | |
522 | 'i\xcc\x87'), | |
523 | # 3.5 Case folding multibyte U+0143 U+037A. | |
524 | ('\xc5\x83\xcd\xba', | |
525 | '\xc5\x84 \xce\xb9'), | |
526 | # 3.6 Case folding U+2121 U+33C6 U+1D7BB. | |
527 | # XXX: skip this as it fails in UCS-2 mode | |
528 | #('\xe2\x84\xa1\xe3\x8f\x86\xf0\x9d\x9e\xbb', | |
529 | # 'telc\xe2\x88\x95kg\xcf\x83'), | |
530 | (None, None), | |
531 | # 3.7 Normalization of U+006a U+030c U+00A0 U+00AA. | |
532 | ('j\xcc\x8c\xc2\xa0\xc2\xaa', | |
533 | '\xc7\xb0 a'), | |
534 | # 3.8 Case folding U+1FB7 and normalization. | |
535 | ('\xe1\xbe\xb7', | |
536 | '\xe1\xbe\xb6\xce\xb9'), | |
537 | # 3.9 Self-reverting case folding U+01F0 and normalization. | |
538 | # The original test case is bogus, it says `\xc7\xf0' | |
539 | ('\xc7\xb0', | |
540 | '\xc7\xb0'), | |
541 | # 3.10 Self-reverting case folding U+0390 and normalization. | |
542 | ('\xce\x90', | |
543 | '\xce\x90'), | |
544 | # 3.11 Self-reverting case folding U+03B0 and normalization. | |
545 | ('\xce\xb0', | |
546 | '\xce\xb0'), | |
547 | # 3.12 Self-reverting case folding U+1E96 and normalization. | |
548 | ('\xe1\xba\x96', | |
549 | '\xe1\xba\x96'), | |
550 | # 3.13 Self-reverting case folding U+1F56 and normalization. | |
551 | ('\xe1\xbd\x96', | |
552 | '\xe1\xbd\x96'), | |
553 | # 3.14 ASCII space character U+0020. | |
554 | (' ', | |
555 | ' '), | |
556 | # 3.15 Non-ASCII 8bit space character U+00A0. | |
557 | ('\xc2\xa0', | |
558 | ' '), | |
559 | # 3.16 Non-ASCII multibyte space character U+1680. | |
560 | ('\xe1\x9a\x80', | |
561 | None), | |
562 | # 3.17 Non-ASCII multibyte space character U+2000. | |
563 | ('\xe2\x80\x80', | |
564 | ' '), | |
565 | # 3.18 Zero Width Space U+200b. | |
566 | ('\xe2\x80\x8b', | |
567 | ''), | |
568 | # 3.19 Non-ASCII multibyte space character U+3000. | |
569 | ('\xe3\x80\x80', | |
570 | ' '), | |
571 | # 3.20 ASCII control characters U+0010 U+007F. | |
572 | ('\x10\x7f', | |
573 | '\x10\x7f'), | |
574 | # 3.21 Non-ASCII 8bit control character U+0085. | |
575 | ('\xc2\x85', | |
576 | None), | |
577 | # 3.22 Non-ASCII multibyte control character U+180E. | |
578 | ('\xe1\xa0\x8e', | |
579 | None), | |
580 | # 3.23 Zero Width No-Break Space U+FEFF. | |
581 | ('\xef\xbb\xbf', | |
582 | ''), | |
583 | # 3.24 Non-ASCII control character U+1D175. | |
584 | ('\xf0\x9d\x85\xb5', | |
585 | None), | |
586 | # 3.25 Plane 0 private use character U+F123. | |
587 | ('\xef\x84\xa3', | |
588 | None), | |
589 | # 3.26 Plane 15 private use character U+F1234. | |
590 | ('\xf3\xb1\x88\xb4', | |
591 | None), | |
592 | # 3.27 Plane 16 private use character U+10F234. | |
593 | ('\xf4\x8f\x88\xb4', | |
594 | None), | |
595 | # 3.28 Non-character code point U+8FFFE. | |
596 | ('\xf2\x8f\xbf\xbe', | |
597 | None), | |
598 | # 3.29 Non-character code point U+10FFFF. | |
599 | ('\xf4\x8f\xbf\xbf', | |
600 | None), | |
601 | # 3.30 Surrogate code U+DF42. | |
602 | ('\xed\xbd\x82', | |
603 | None), | |
604 | # 3.31 Non-plain text character U+FFFD. | |
605 | ('\xef\xbf\xbd', | |
606 | None), | |
607 | # 3.32 Ideographic description character U+2FF5. | |
608 | ('\xe2\xbf\xb5', | |
609 | None), | |
610 | # 3.33 Display property character U+0341. | |
611 | ('\xcd\x81', | |
612 | '\xcc\x81'), | |
613 | # 3.34 Left-to-right mark U+200E. | |
614 | ('\xe2\x80\x8e', | |
615 | None), | |
616 | # 3.35 Deprecated U+202A. | |
617 | ('\xe2\x80\xaa', | |
618 | None), | |
619 | # 3.36 Language tagging character U+E0001. | |
620 | ('\xf3\xa0\x80\x81', | |
621 | None), | |
622 | # 3.37 Language tagging character U+E0042. | |
623 | ('\xf3\xa0\x81\x82', | |
624 | None), | |
625 | # 3.38 Bidi: RandALCat character U+05BE and LCat characters. | |
626 | ('foo\xd6\xbebar', | |
627 | None), | |
628 | # 3.39 Bidi: RandALCat character U+FD50 and LCat characters. | |
629 | ('foo\xef\xb5\x90bar', | |
630 | None), | |
631 | # 3.40 Bidi: RandALCat character U+FB38 and LCat characters. | |
632 | ('foo\xef\xb9\xb6bar', | |
633 | 'foo \xd9\x8ebar'), | |
634 | # 3.41 Bidi: RandALCat without trailing RandALCat U+0627 U+0031. | |
635 | ('\xd8\xa71', | |
636 | None), | |
637 | # 3.42 Bidi: RandALCat character U+0627 U+0031 U+0628. | |
638 | ('\xd8\xa71\xd8\xa8', | |
639 | '\xd8\xa71\xd8\xa8'), | |
640 | # 3.43 Unassigned code point U+E0002. | |
641 | # Skip this test as we allow unassigned | |
642 | #('\xf3\xa0\x80\x82', | |
643 | # None), | |
644 | (None, None), | |
645 | # 3.44 Larger test (shrinking). | |
646 | # Original test case reads \xc3\xdf | |
647 | ('X\xc2\xad\xc3\x9f\xc4\xb0\xe2\x84\xa1j\xcc\x8c\xc2\xa0\xc2' | |
648 | '\xaa\xce\xb0\xe2\x80\x80', | |
649 | 'xssi\xcc\x87tel\xc7\xb0 a\xce\xb0 '), | |
650 | # 3.45 Larger test (expanding). | |
651 | # Original test case reads \xc3\x9f | |
652 | ('X\xc3\x9f\xe3\x8c\x96\xc4\xb0\xe2\x84\xa1\xe2\x92\x9f\xe3\x8c' | |
653 | '\x80', | |
654 | 'xss\xe3\x82\xad\xe3\x83\xad\xe3\x83\xa1\xe3\x83\xbc\xe3' | |
655 | '\x83\x88\xe3\x83\xabi\xcc\x87tel\x28d\x29\xe3\x82' | |
656 | '\xa2\xe3\x83\x91\xe3\x83\xbc\xe3\x83\x88') | |
657 | ] | |
658 | ||
659 | ||
660 | class NameprepTest(unittest.TestCase): | |
661 | def test_nameprep(self): | |
662 | from encodings.idna import nameprep | |
663 | for pos, (orig, prepped) in enumerate(nameprep_tests): | |
664 | if orig is None: | |
665 | # Skipped | |
666 | continue | |
667 | # The Unicode strings are given in UTF-8 | |
668 | orig = unicode(orig, "utf-8") | |
669 | if prepped is None: | |
670 | # Input contains prohibited characters | |
671 | self.assertRaises(UnicodeError, nameprep, orig) | |
672 | else: | |
673 | prepped = unicode(prepped, "utf-8") | |
674 | try: | |
675 | self.assertEquals(nameprep(orig), prepped) | |
676 | except Exception,e: | |
677 | raise test_support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) | |
678 | ||
679 | class CodecTest(unittest.TestCase): | |
680 | def test_builtin(self): | |
681 | self.assertEquals(unicode("python.org", "idna"), u"python.org") | |
682 | ||
683 | def test_stream(self): | |
684 | import StringIO | |
685 | r = codecs.getreader("idna")(StringIO.StringIO("abc")) | |
686 | r.read(3) | |
687 | self.assertEquals(r.read(), u"") | |
688 | ||
689 | class CodecsModuleTest(unittest.TestCase): | |
690 | ||
691 | def test_decode(self): | |
692 | self.assertEquals(codecs.decode('\xe4\xf6\xfc', 'latin-1'), | |
693 | u'\xe4\xf6\xfc') | |
694 | self.assertRaises(TypeError, codecs.decode) | |
695 | self.assertEquals(codecs.decode('abc'), u'abc') | |
696 | self.assertRaises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii') | |
697 | ||
698 | def test_encode(self): | |
699 | self.assertEquals(codecs.encode(u'\xe4\xf6\xfc', 'latin-1'), | |
700 | '\xe4\xf6\xfc') | |
701 | self.assertRaises(TypeError, codecs.encode) | |
702 | self.assertEquals(codecs.encode(u'abc'), 'abc') | |
703 | self.assertRaises(UnicodeEncodeError, codecs.encode, u'\xffff', 'ascii') | |
704 | ||
705 | def test_register(self): | |
706 | self.assertRaises(TypeError, codecs.register) | |
707 | ||
708 | def test_lookup(self): | |
709 | self.assertRaises(TypeError, codecs.lookup) | |
710 | self.assertRaises(LookupError, codecs.lookup, "__spam__") | |
711 | ||
712 | class StreamReaderTest(unittest.TestCase): | |
713 | ||
714 | def setUp(self): | |
715 | self.reader = codecs.getreader('utf-8') | |
716 | self.stream = StringIO.StringIO('\xed\x95\x9c\n\xea\xb8\x80') | |
717 | ||
718 | def test_readlines(self): | |
719 | f = self.reader(self.stream) | |
720 | self.assertEquals(f.readlines(), [u'\ud55c\n', u'\uae00']) | |
721 | ||
722 | class Str2StrTest(unittest.TestCase): | |
723 | ||
724 | def test_read(self): | |
725 | sin = "\x80".encode("base64_codec") | |
726 | reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin)) | |
727 | sout = reader.read() | |
728 | self.assertEqual(sout, "\x80") | |
729 | self.assert_(isinstance(sout, str)) | |
730 | ||
731 | def test_readline(self): | |
732 | sin = "\x80".encode("base64_codec") | |
733 | reader = codecs.getreader("base64_codec")(StringIO.StringIO(sin)) | |
734 | sout = reader.readline() | |
735 | self.assertEqual(sout, "\x80") | |
736 | self.assert_(isinstance(sout, str)) | |
737 | ||
738 | def test_main(): | |
739 | test_support.run_unittest( | |
740 | UTF16Test, | |
741 | UTF16LETest, | |
742 | UTF16BETest, | |
743 | UTF8Test, | |
744 | EscapeDecodeTest, | |
745 | RecodingTest, | |
746 | PunycodeTest, | |
747 | UnicodeInternalTest, | |
748 | NameprepTest, | |
749 | CodecTest, | |
750 | CodecsModuleTest, | |
751 | StreamReaderTest, | |
752 | Str2StrTest | |
753 | ) | |
754 | ||
755 | ||
756 | if __name__ == "__main__": | |
757 | test_main() |