Commit | Line | Data |
---|---|---|
920dae64 AT |
1 | .\" Automatically generated by Pod::Man v1.37, Pod::Parser v1.32 |
2 | .\" | |
3 | .\" Standard preamble: | |
4 | .\" ======================================================================== | |
5 | .de Sh \" Subsection heading | |
6 | .br | |
7 | .if t .Sp | |
8 | .ne 5 | |
9 | .PP | |
10 | \fB\\$1\fR | |
11 | .PP | |
12 | .. | |
13 | .de Sp \" Vertical space (when we can't use .PP) | |
14 | .if t .sp .5v | |
15 | .if n .sp | |
16 | .. | |
17 | .de Vb \" Begin verbatim text | |
18 | .ft CW | |
19 | .nf | |
20 | .ne \\$1 | |
21 | .. | |
22 | .de Ve \" End verbatim text | |
23 | .ft R | |
24 | .fi | |
25 | .. | |
26 | .\" Set up some character translations and predefined strings. \*(-- will | |
27 | .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left | |
28 | .\" double quote, and \*(R" will give a right double quote. | will give a | |
29 | .\" real vertical bar. \*(C+ will give a nicer C++. Capital omega is used to | |
30 | .\" do unbreakable dashes and therefore won't be available. \*(C` and \*(C' | |
31 | .\" expand to `' in nroff, nothing in troff, for use with C<>. | |
32 | .tr \(*W-|\(bv\*(Tr | |
33 | .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' | |
34 | .ie n \{\ | |
35 | . ds -- \(*W- | |
36 | . ds PI pi | |
37 | . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch | |
38 | . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch | |
39 | . ds L" "" | |
40 | . ds R" "" | |
41 | . ds C` "" | |
42 | . ds C' "" | |
43 | 'br\} | |
44 | .el\{\ | |
45 | . ds -- \|\(em\| | |
46 | . ds PI \(*p | |
47 | . ds L" `` | |
48 | . ds R" '' | |
49 | 'br\} | |
50 | .\" | |
51 | .\" If the F register is turned on, we'll generate index entries on stderr for | |
52 | .\" titles (.TH), headers (.SH), subsections (.Sh), items (.Ip), and index | |
53 | .\" entries marked with X<> in POD. Of course, you'll have to process the | |
54 | .\" output yourself in some meaningful fashion. | |
55 | .if \nF \{\ | |
56 | . de IX | |
57 | . tm Index:\\$1\t\\n%\t"\\$2" | |
58 | .. | |
59 | . nr % 0 | |
60 | . rr F | |
61 | .\} | |
62 | .\" | |
63 | .\" For nroff, turn off justification. Always turn off hyphenation; it makes | |
64 | .\" way too many mistakes in technical documents. | |
65 | .hy 0 | |
66 | .if n .na | |
67 | .\" | |
68 | .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). | |
69 | .\" Fear. Run. Save yourself. No user-serviceable parts. | |
70 | . \" fudge factors for nroff and troff | |
71 | .if n \{\ | |
72 | . ds #H 0 | |
73 | . ds #V .8m | |
74 | . ds #F .3m | |
75 | . ds #[ \f1 | |
76 | . ds #] \fP | |
77 | .\} | |
78 | .if t \{\ | |
79 | . ds #H ((1u-(\\\\n(.fu%2u))*.13m) | |
80 | . ds #V .6m | |
81 | . ds #F 0 | |
82 | . ds #[ \& | |
83 | . ds #] \& | |
84 | .\} | |
85 | . \" simple accents for nroff and troff | |
86 | .if n \{\ | |
87 | . ds ' \& | |
88 | . ds ` \& | |
89 | . ds ^ \& | |
90 | . ds , \& | |
91 | . ds ~ ~ | |
92 | . ds / | |
93 | .\} | |
94 | .if t \{\ | |
95 | . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" | |
96 | . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' | |
97 | . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' | |
98 | . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' | |
99 | . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' | |
100 | . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' | |
101 | .\} | |
102 | . \" troff and (daisy-wheel) nroff accents | |
103 | .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' | |
104 | .ds 8 \h'\*(#H'\(*b\h'-\*(#H' | |
105 | .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] | |
106 | .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' | |
107 | .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' | |
108 | .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] | |
109 | .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] | |
110 | .ds ae a\h'-(\w'a'u*4/10)'e | |
111 | .ds Ae A\h'-(\w'A'u*4/10)'E | |
112 | . \" corrections for vroff | |
113 | .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' | |
114 | .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' | |
115 | . \" for low resolution devices (crt and lpr) | |
116 | .if \n(.H>23 .if \n(.V>19 \ | |
117 | \{\ | |
118 | . ds : e | |
119 | . ds 8 ss | |
120 | . ds o a | |
121 | . ds d- d\h'-1'\(ga | |
122 | . ds D- D\h'-1'\(hy | |
123 | . ds th \o'bp' | |
124 | . ds Th \o'LP' | |
125 | . ds ae ae | |
126 | . ds Ae AE | |
127 | .\} | |
128 | .rm #[ #] #H #V #F C | |
129 | .\" ======================================================================== | |
130 | .\" | |
131 | .IX Title "Encode::PerlIO 3" | |
132 | .TH Encode::PerlIO 3 "2001-09-21" "perl v5.8.8" "Perl Programmers Reference Guide" | |
133 | .SH "NAME" | |
134 | Encode::PerlIO \-\- a detailed document on Encode and PerlIO | |
135 | .SH "Overview" | |
136 | .IX Header "Overview" | |
137 | It is very common to want to do encoding transformations when | |
138 | reading or writing files, network connections, pipes etc. | |
139 | If Perl is configured to use the new 'perlio' \s-1IO\s0 system then | |
140 | \&\f(CW\*(C`Encode\*(C'\fR provides a \*(L"layer\*(R" (see PerlIO) which can transform | |
141 | data as it is read or written. | |
142 | .PP | |
143 | Here is how the blind poet would modernise the encoding: | |
144 | .PP | |
145 | .Vb 7 | |
146 | \& use Encode; | |
147 | \& open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek'); | |
148 | \& open(my $utf8,'>:utf8','iliad.utf8'); | |
149 | \& my @epic = <$iliad>; | |
150 | \& print $utf8 @epic; | |
151 | \& close($utf8); | |
152 | \& close($illiad); | |
153 | .Ve | |
154 | .PP | |
155 | In addition, the new \s-1IO\s0 system can also be configured to read/write | |
156 | \&\s-1UTF\-8\s0 encoded characters (as noted above, this is efficient): | |
157 | .PP | |
158 | .Vb 2 | |
159 | \& open(my $fh,'>:utf8','anything'); | |
160 | \& print $fh "Any \ex{0021} string \eN{SMILEY FACE}\en"; | |
161 | .Ve | |
162 | .PP | |
163 | Either of the above forms of \*(L"layer\*(R" specifications can be made the default | |
164 | for a lexical scope with the \f(CW\*(C`use open ...\*(C'\fR pragma. See open. | |
165 | .PP | |
166 | Once a handle is open, its layers can be altered using \f(CW\*(C`binmode\*(C'\fR. | |
167 | .PP | |
168 | Without any such configuration, or if Perl itself is built using the | |
169 | system's own \s-1IO\s0, then write operations assume that the file handle | |
170 | accepts only \fIbytes\fR and will \f(CW\*(C`die\*(C'\fR if a character larger than 255 is | |
171 | written to the handle. When reading, each octet from the handle becomes | |
172 | a byte\-in\-a\-character. Note that this default is the same behaviour | |
173 | as bytes-only languages (including Perl before v5.6) would have, | |
174 | and is sufficient to handle native 8\-bit encodings e.g. iso\-8859\-1, | |
175 | \&\s-1EBCDIC\s0 etc. and any legacy mechanisms for handling other encodings | |
176 | and binary data. | |
177 | .PP | |
178 | In other cases, it is the program's responsibility to transform | |
179 | characters into bytes using the \s-1API\s0 above before doing writes, and to | |
180 | transform the bytes read from a handle into characters before doing | |
181 | \&\*(L"character operations\*(R" (e.g. \f(CW\*(C`lc\*(C'\fR, \f(CW\*(C`/\eW+/\*(C'\fR, ...). | |
182 | .PP | |
183 | You can also use PerlIO to convert larger amounts of data you don't | |
184 | want to bring into memory. For example, to convert between \s-1ISO\-8859\-1\s0 | |
185 | (Latin 1) and \s-1UTF\-8\s0 (or UTF-EBCDIC in \s-1EBCDIC\s0 machines): | |
186 | .PP | |
187 | .Vb 3 | |
188 | \& open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!; | |
189 | \& open(G, ">:utf8", "data.utf") or die $!; | |
190 | \& while (<F>) { print G } | |
191 | .Ve | |
192 | .PP | |
193 | .Vb 2 | |
194 | \& # Could also do "print G <F>" but that would pull | |
195 | \& # the whole file into memory just to write it out again. | |
196 | .Ve | |
197 | .PP | |
198 | More examples: | |
199 | .PP | |
200 | .Vb 3 | |
201 | \& open(my $f, "<:encoding(cp1252)") | |
202 | \& open(my $g, ">:encoding(iso-8859-2)") | |
203 | \& open(my $h, ">:encoding(latin9)") # iso-8859-15 | |
204 | .Ve | |
205 | .PP | |
206 | See also encoding for how to change the default encoding of the | |
207 | data in your script. | |
208 | .SH "How does it work?" | |
209 | .IX Header "How does it work?" | |
210 | Here is a crude diagram of how filehandle, PerlIO, and Encode | |
211 | interact. | |
212 | .PP | |
213 | .Vb 3 | |
214 | \& filehandle <-> PerlIO PerlIO <-> scalar (read/printed) | |
215 | \& \e / | |
216 | \& Encode | |
217 | .Ve | |
218 | .PP | |
219 | When PerlIO receives data from either direction, it fills a buffer | |
220 | (currently with 1024 bytes) and passes the buffer to Encode. | |
221 | Encode tries to convert the valid part and passes it back to PerlIO, | |
222 | leaving invalid parts (usually a partial character) in the buffer. | |
223 | PerlIO then appends more data to the buffer, calls Encode again, | |
224 | and so on until the data stream ends. | |
225 | .PP | |
226 | To do so, PerlIO always calls (de|en)code methods with \s-1CHECK\s0 set to 1. | |
227 | This ensures that the method stops at the right place when it | |
228 | encounters partial character. The following is what happens when | |
229 | PerlIO and Encode tries to encode (from utf8) more than 1024 bytes | |
230 | and the buffer boundary happens to be in the middle of a character. | |
231 | .PP | |
232 | .Vb 5 | |
233 | \& A B C .... ~ \ex{3000} .... | |
234 | \& 41 42 43 .... 7E e3 80 80 .... | |
235 | \& <- buffer ---------------> | |
236 | \& << encoded >>>>>>>>>> | |
237 | \& <- next buffer ------ | |
238 | .Ve | |
239 | .PP | |
240 | Encode converts from the beginning to \ex7E, leaving \exe3 in the buffer | |
241 | because it is invalid (partial character). | |
242 | .PP | |
243 | Unfortunately, this scheme does not work well with escape-based | |
244 | encodings such as \s-1ISO\-2022\-JP\s0. | |
245 | .SH "Line Buffering" | |
246 | .IX Header "Line Buffering" | |
247 | Now let's see what happens when you try to decode from \s-1ISO\-2022\-JP\s0 and | |
248 | the buffer ends in the middle of a character. | |
249 | .PP | |
250 | .Vb 5 | |
251 | \& JIS208-ESC \ex{5f3e} | |
252 | \& A B C .... ~ \ee $ B |DAN | .... | |
253 | \& 41 42 43 .... 7E 1b 24 41 43 46 .... | |
254 | \& <- buffer ---------------------------> | |
255 | \& << encoded >>>>>>>>>>>>>>>>>>>>>>> | |
256 | .Ve | |
257 | .PP | |
258 | As you see, the next buffer begins with \ex43. But \ex43 is 'C' in | |
259 | \&\s-1ASCII\s0, which is wrong in this case because we are now in \s-1JISX\s0 0208 | |
260 | area so it has to convert \ex43\ex46, not \ex43. Unlike utf8 and \s-1EUC\s0, | |
261 | in escape-based encodings you can't tell if a given octet is a whole | |
262 | character or just part of it. | |
263 | .PP | |
264 | Fortunately PerlIO also supports line buffer if you tell PerlIO to use | |
265 | one instead of fixed buffer. Since \s-1ISO\-2022\-JP\s0 is guaranteed to revert to \s-1ASCII\s0 at the end of the line, partial | |
266 | character will never happen when line buffer is used. | |
267 | .PP | |
268 | To tell PerlIO to use line buffer, implement \->needs_lines method | |
269 | for your encoding object. See Encode::Encoding for details. | |
270 | .PP | |
271 | Thanks to these efforts most encodings that come with Encode support | |
272 | PerlIO but that still leaves following encodings. | |
273 | .PP | |
274 | .Vb 4 | |
275 | \& iso-2022-kr | |
276 | \& MIME-B | |
277 | \& MIME-Header | |
278 | \& MIME-Q | |
279 | .Ve | |
280 | .PP | |
281 | Fortunately iso\-2022\-kr is hardly used (according to Jungshik) and | |
282 | MIME\-* are very unlikely to be fed to PerlIO because they are for mail | |
283 | headers. See Encode::MIME::Header for details. | |
284 | .Sh "How can I tell whether my encoding fully supports PerlIO ?" | |
285 | .IX Subsection "How can I tell whether my encoding fully supports PerlIO ?" | |
286 | As of this writing, any encoding whose class belongs to Encode::XS and | |
287 | Encode::Unicode works. The Encode module has a \f(CW\*(C`perlio_ok\*(C'\fR method | |
288 | which you can use before applying PerlIO encoding to the filehandle. | |
289 | Here is an example: | |
290 | .PP | |
291 | .Vb 7 | |
292 | \& my $use_perlio = perlio_ok($enc); | |
293 | \& my $layer = $use_perlio ? "<:raw" : "<:encoding($enc)"; | |
294 | \& open my $fh, $layer, $file or die "$file : $!"; | |
295 | \& while(<$fh>){ | |
296 | \& $_ = decode($enc, $_) unless $use_perlio; | |
297 | \& # .... | |
298 | \& } | |
299 | .Ve | |
300 | .SH "SEE ALSO" | |
301 | .IX Header "SEE ALSO" | |
302 | Encode::Encoding, | |
303 | Encode::Supported, | |
304 | Encode::PerlIO, | |
305 | encoding, | |
306 | perlebcdic, | |
307 | \&\*(L"open\*(R" in perlfunc, | |
308 | perlunicode, | |
309 | utf8, | |
310 | the Perl Unicode Mailing List <perl\-unicode@perl.org> |