| 1 | .\" Automatically generated by Pod::Man v1.34, Pod::Parser v1.13 |
| 2 | .\" |
| 3 | .\" Standard preamble: |
| 4 | .\" ======================================================================== |
| 5 | .de Sh \" Subsection heading |
| 6 | .br |
| 7 | .if t .Sp |
| 8 | .ne 5 |
| 9 | .PP |
| 10 | \fB\\$1\fR |
| 11 | .PP |
| 12 | .. |
| 13 | .de Sp \" Vertical space (when we can't use .PP) |
| 14 | .if t .sp .5v |
| 15 | .if n .sp |
| 16 | .. |
| 17 | .de Vb \" Begin verbatim text |
| 18 | .ft CW |
| 19 | .nf |
| 20 | .ne \\$1 |
| 21 | .. |
| 22 | .de Ve \" End verbatim text |
| 23 | .ft R |
| 24 | .fi |
| 25 | .. |
| 26 | .\" Set up some character translations and predefined strings. \*(-- will |
| 27 | .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left |
| 28 | .\" double quote, and \*(R" will give a right double quote. | will give a |
| 29 | .\" real vertical bar. \*(C+ will give a nicer C++. Capital omega is used to |
| 30 | .\" do unbreakable dashes and therefore won't be available. \*(C` and \*(C' |
| 31 | .\" expand to `' in nroff, nothing in troff, for use with C<>. |
| 32 | .tr \(*W-|\(bv\*(Tr |
| 33 | .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' |
| 34 | .ie n \{\ |
| 35 | . ds -- \(*W- |
| 36 | . ds PI pi |
| 37 | . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch |
| 38 | . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch |
| 39 | . ds L" "" |
| 40 | . ds R" "" |
| 41 | . ds C` "" |
| 42 | . ds C' "" |
| 43 | 'br\} |
| 44 | .el\{\ |
| 45 | . ds -- \|\(em\| |
| 46 | . ds PI \(*p |
| 47 | . ds L" `` |
| 48 | . ds R" '' |
| 49 | 'br\} |
| 50 | .\" |
| 51 | .\" If the F register is turned on, we'll generate index entries on stderr for |
| 52 | .\" titles (.TH), headers (.SH), subsections (.Sh), items (.Ip), and index |
| 53 | .\" entries marked with X<> in POD. Of course, you'll have to process the |
| 54 | .\" output yourself in some meaningful fashion. |
| 55 | .if \nF \{\ |
| 56 | . de IX |
| 57 | . tm Index:\\$1\t\\n%\t"\\$2" |
| 58 | .. |
| 59 | . nr % 0 |
| 60 | . rr F |
| 61 | .\} |
| 62 | .\" |
| 63 | .\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 64 | .\" way too many mistakes in technical documents. |
| 65 | .hy 0 |
| 66 | .if n .na |
| 67 | .\" |
| 68 | .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). |
| 69 | .\" Fear. Run. Save yourself. No user-serviceable parts. |
| 70 | . \" fudge factors for nroff and troff |
| 71 | .if n \{\ |
| 72 | . ds #H 0 |
| 73 | . ds #V .8m |
| 74 | . ds #F .3m |
| 75 | . ds #[ \f1 |
| 76 | . ds #] \fP |
| 77 | .\} |
| 78 | .if t \{\ |
| 79 | . ds #H ((1u-(\\\\n(.fu%2u))*.13m) |
| 80 | . ds #V .6m |
| 81 | . ds #F 0 |
| 82 | . ds #[ \& |
| 83 | . ds #] \& |
| 84 | .\} |
| 85 | . \" simple accents for nroff and troff |
| 86 | .if n \{\ |
| 87 | . ds ' \& |
| 88 | . ds ` \& |
| 89 | . ds ^ \& |
| 90 | . ds , \& |
| 91 | . ds ~ ~ |
| 92 | . ds / |
| 93 | .\} |
| 94 | .if t \{\ |
| 95 | . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" |
| 96 | . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' |
| 97 | . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' |
| 98 | . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' |
| 99 | . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' |
| 100 | . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' |
| 101 | .\} |
| 102 | . \" troff and (daisy-wheel) nroff accents |
| 103 | .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' |
| 104 | .ds 8 \h'\*(#H'\(*b\h'-\*(#H' |
| 105 | .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] |
| 106 | .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' |
| 107 | .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' |
| 108 | .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] |
| 109 | .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] |
| 110 | .ds ae a\h'-(\w'a'u*4/10)'e |
| 111 | .ds Ae A\h'-(\w'A'u*4/10)'E |
| 112 | . \" corrections for vroff |
| 113 | .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' |
| 114 | .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' |
| 115 | . \" for low resolution devices (crt and lpr) |
| 116 | .if \n(.H>23 .if \n(.V>19 \ |
| 117 | \{\ |
| 118 | . ds : e |
| 119 | . ds 8 ss |
| 120 | . ds o a |
| 121 | . ds d- d\h'-1'\(ga |
| 122 | . ds D- D\h'-1'\(hy |
| 123 | . ds th \o'bp' |
| 124 | . ds Th \o'LP' |
| 125 | . ds ae ae |
| 126 | . ds Ae AE |
| 127 | .\} |
| 128 | .rm #[ #] #H #V #F C |
| 129 | .\" ======================================================================== |
| 130 | .\" |
| 131 | .IX Title "Encode::Unicode 3" |
| 132 | .TH Encode::Unicode 3 "2002-06-01" "perl v5.8.0" "Perl Programmers Reference Guide" |
| 133 | .SH "NAME" |
| 134 | Encode::Unicode \-\- Various Unicode Transformation Formats |
| 135 | .SH "SYNOPSIS" |
| 136 | .IX Header "SYNOPSIS" |
| 137 | .Vb 3 |
| 138 | \& use Encode qw/encode decode/; |
| 139 | \& $ucs2 = encode("UCS-2BE", $utf8); |
| 140 | \& $utf8 = decode("UCS-2BE", $ucs2); |
| 141 | .Ve |
| 142 | .SH "ABSTRACT" |
| 143 | .IX Header "ABSTRACT" |
| 144 | This module implements all Character Encoding Schemes of Unicode that |
| 145 | are officially documented by Unicode Consortium (except, of course, |
| 146 | for \s-1UTF\-8\s0, which is a native format in perl). |
| 147 | .IP "<http://www.unicode.org/glossary/> says:" 4 |
| 148 | .IX Item "<http://www.unicode.org/glossary/> says:" |
| 149 | \&\fICharacter Encoding Scheme\fR A character encoding form plus byte |
| 150 | serialization. There are seven character encoding schemes in Unicode: |
| 151 | \&\s-1UTF\-8\s0, \s-1UTF\-16\s0, \s-1UTF\-16BE\s0, \s-1UTF\-16LE\s0, \s-1UTF\-32\s0 (\s-1UCS\-4\s0), \s-1UTF\-32BE\s0 (\s-1UCS\-4BE\s0) and |
| 152 | \&\s-1UTF\-32LE\s0 (\s-1UCS\-4LE\s0). |
| 153 | .IP "Quick Reference" 4 |
| 154 | .IX Item "Quick Reference" |
| 155 | .Vb 13 |
| 156 | \& Decodes from ord(N) Encodes chr(N) to... |
| 157 | \& octet/char BOM S.P d800-dfff ord > 0xffff \ex{1abcd} == |
| 158 | \& ---------------+-----------------+------------------------------ |
| 159 | \& UCS-2BE 2 N N is bogus Not Available |
| 160 | \& UCS-2LE 2 N N bogus Not Available |
| 161 | \& UTF-16 2/4 Y Y is S.P S.P BE/LE |
| 162 | \& UTF-16BE 2/4 N Y S.P S.P 0xd82a,0xdfcd |
| 163 | \& UTF-16LE 2 N Y S.P S.P 0x2ad8,0xcddf |
| 164 | \& UTF-32 4 Y - is bogus As is BE/LE |
| 165 | \& UTF-32BE 4 N - bogus As is 0x0001abcd |
| 166 | \& UTF-32LE 4 N - bogus As is 0xcdab0100 |
| 167 | \& UTF-8 1-4 - - bogus >= 4 octets \exf0\ex9a\eaf\e8d |
| 168 | \& ---------------+-----------------+------------------------------ |
| 169 | .Ve |
| 170 | .SH "Size, Endianness, and BOM" |
| 171 | .IX Header "Size, Endianness, and BOM" |
| 172 | You can categorize these \s-1CES\s0 by 3 criteria: size of each character, |
| 173 | endianness, and Byte Order Mark. |
| 174 | .Sh "by size" |
| 175 | .IX Subsection "by size" |
| 176 | \&\s-1UCS\-2\s0 is a fixed-length encoding with each character taking 16 bits. |
| 177 | It \fBdoes not\fR support \fIsurrogate pairs\fR. When a surrogate pair |
| 178 | is encountered during \fIdecode()\fR, its place is filled with \ex{\s-1FFFD\s0} |
| 179 | if \fI\s-1CHECK\s0\fR is 0, or the routine croaks if \fI\s-1CHECK\s0\fR is 1. When a |
| 180 | character whose ord value is larger than 0xFFFF is encountered, |
| 181 | its place is filled with \ex{\s-1FFFD\s0} if \fI\s-1CHECK\s0\fR is 0, or the routine |
| 182 | croaks if \fI\s-1CHECK\s0\fR is 1. |
| 183 | .PP |
| 184 | \&\s-1UTF\-16\s0 is almost the same as \s-1UCS\-2\s0 but it supports \fIsurrogate pairs\fR. |
| 185 | When it encounters a high surrogate (0xD800\-0xDBFF), it fetches the |
| 186 | following low surrogate (0xDC00\-0xDFFF) and \f(CW\*(C`desurrogate\*(C'\fRs them to |
| 187 | form a character. Bogus surrogates result in death. When \ex{10000} |
| 188 | or above is encountered during \fIencode()\fR, it \f(CW\*(C`ensurrogate\*(C'\fRs them and |
| 189 | pushes the surrogate pair to the output stream. |
| 190 | .PP |
| 191 | \&\s-1UTF\-32\s0 (\s-1UCS\-4\s0) is a fixed-length encoding with each character taking 32 bits. |
| 192 | Since it is 32\-bit, there is no need for \fIsurrogate pairs\fR. |
| 193 | .Sh "by endianness" |
| 194 | .IX Subsection "by endianness" |
| 195 | The first (and now failed) goal of Unicode was to map all character |
| 196 | repertoires into a fixed-length integer so that programmers are happy. |
| 197 | Since each character is either a \fIshort\fR or \fIlong\fR in C, you have to |
| 198 | pay attention to the endianness of each platform when you pass data |
| 199 | to one another. |
| 200 | .PP |
| 201 | Anything marked as \s-1BE\s0 is Big Endian (or network byte order) and \s-1LE\s0 is |
| 202 | Little Endian (aka \s-1VAX\s0 byte order). For anything not marked either |
| 203 | \&\s-1BE\s0 or \s-1LE\s0, a character called Byte Order Mark (\s-1BOM\s0) indicating the |
| 204 | endianness is prepended to the string. |
| 205 | .IP "\s-1BOM\s0 as integer when fetched in network byte order" 4 |
| 206 | .IX Item "BOM as integer when fetched in network byte order" |
| 207 | .Vb 5 |
| 208 | \& 16 32 bits/char |
| 209 | \& ------------------------- |
| 210 | \& BE 0xFeFF 0x0000FeFF |
| 211 | \& LE 0xFFeF 0xFFFe0000 |
| 212 | \& ------------------------- |
| 213 | .Ve |
| 214 | .PP |
| 215 | This modules handles the \s-1BOM\s0 as follows. |
| 216 | .IP "\(bu" 4 |
| 217 | When \s-1BE\s0 or \s-1LE\s0 is explicitly stated as the name of encoding, \s-1BOM\s0 is |
| 218 | simply treated as a normal character (\s-1ZERO\s0 \s-1WIDTH\s0 NO-BREAK \s-1SPACE\s0). |
| 219 | .IP "\(bu" 4 |
| 220 | When \s-1BE\s0 or \s-1LE\s0 is omitted during \fIdecode()\fR, it checks if \s-1BOM\s0 is at the |
| 221 | beginning of the string; if one is found, the endianness is set to |
| 222 | what the \s-1BOM\s0 says. If no \s-1BOM\s0 is found, the routine dies. |
| 223 | .IP "\(bu" 4 |
| 224 | When \s-1BE\s0 or \s-1LE\s0 is omitted during \fIencode()\fR, it returns a BE-encoded |
| 225 | string with \s-1BOM\s0 prepended. So when you want to encode a whole text |
| 226 | file, make sure you \fIencode()\fR the whole text at once, not line by line |
| 227 | or each line, not file, will have a \s-1BOM\s0 prepended. |
| 228 | .IP "\(bu" 4 |
| 229 | \&\f(CW\*(C`UCS\-2\*(C'\fR is an exception. Unlike others, this is an alias of \s-1UCS\-2BE\s0. |
| 230 | \&\s-1UCS\-2\s0 is already registered by \s-1IANA\s0 and others that way. |
| 231 | .SH "Surrogate Pairs" |
| 232 | .IX Header "Surrogate Pairs" |
| 233 | To say the least, surrogate pairs were the biggest mistake of the |
| 234 | Unicode Consortium. But according to the late Douglas Adams in \fIThe |
| 235 | Hitchhiker's Guide to the Galaxy\fR Trilogy, \f(CW\*(C`In the beginning the |
| 236 | Universe was created. This has made a lot of people very angry and |
| 237 | been widely regarded as a bad move\*(C'\fR. Their mistake was not of this |
| 238 | magnitude so let's forgive them. |
| 239 | .PP |
| 240 | (I don't dare make any comparison with Unicode Consortium and the |
| 241 | Vogons here ;) Or, comparing Encode to Babel Fish is completely |
| 242 | appropriate \*(-- if you can only stick this into your ear :) |
| 243 | .PP |
| 244 | Surrogate pairs were born when the Unicode Consortium finally |
| 245 | admitted that 16 bits were not big enough to hold all the world's |
| 246 | character repertoires. But they already made \s-1UCS\-2\s0 16\-bit. What |
| 247 | do we do? |
| 248 | .PP |
| 249 | Back then, the range 0xD800\-0xDFFF was not allocated. Let's split |
| 250 | that range in half and use the first half to represent the \f(CW\*(C`upper |
| 251 | half of a character\*(C'\fR and the second half to represent the \f(CW\*(C`lower |
| 252 | half of a character\*(C'\fR. That way, you can represent 1024 * 1024 = |
| 253 | 1048576 more characters. Now we can store character ranges up to |
| 254 | \&\ex{10ffff} even with 16\-bit encodings. This pair of half-character is |
| 255 | now called a \fIsurrogate pair\fR and \s-1UTF\-16\s0 is the name of the encoding |
| 256 | that embraces them. |
| 257 | .PP |
| 258 | Here is a formula to ensurrogate a Unicode character \ex{10000} and |
| 259 | above; |
| 260 | .PP |
| 261 | .Vb 2 |
| 262 | \& $hi = ($uni - 0x10000) / 0x400 + 0xD800; |
| 263 | \& $lo = ($uni - 0x10000) % 0x400 + 0xDC00; |
| 264 | .Ve |
| 265 | .PP |
| 266 | And to desurrogate; |
| 267 | .PP |
| 268 | .Vb 1 |
| 269 | \& $uni = 0x10000 + ($hi - 0xD800) * 0x400 + ($lo - 0xDC00); |
| 270 | .Ve |
| 271 | .PP |
| 272 | Note this move has made \ex{D800}\-\ex{\s-1DFFF\s0} into a forbidden zone but |
| 273 | perl does not prohibit the use of characters within this range. To perl, |
| 274 | every one of \ex{0000_0000} up to \ex{ffff_ffff} (*) is \fIa character\fR. |
| 275 | .PP |
| 276 | .Vb 2 |
| 277 | \& (*) or \ex{ffff_ffff_ffff_ffff} if your perl is compiled with 64-bit |
| 278 | \& integer support! |
| 279 | .Ve |
| 280 | .SH "SEE ALSO" |
| 281 | .IX Header "SEE ALSO" |
| 282 | Encode, <http://www.unicode.org/glossary/>, |
| 283 | <http://www.unicode.org/unicode/faq/utf_bom.html>, |
| 284 | .PP |
| 285 | \&\s-1RFC\s0 2781 <http://rfc.net/rfc2781.html>, |
| 286 | .PP |
| 287 | The whole Unicode standard <http://www.unicode.org/unicode/uni2book/u2.html> |
| 288 | .PP |
| 289 | Ch. 15, pp. 403 of \f(CW\*(C`Programming Perl (3rd Edition)\*(C'\fR |
| 290 | by Larry Wall, Tom Christiansen, Jon Orwant; |
| 291 | O'Reilly & Associates; \s-1ISBN\s0 0\-596\-00027\-8 |