Annotation of embedaddon/php/ext/fileinfo/libmagic/encoding.c, revision 1.1.1.1
1.1 misho 1: /*
2: * Copyright (c) Ian F. Darwin 1986-1995.
3: * Software written by Ian F. Darwin and others;
4: * maintained 1995-present by Christos Zoulas and others.
5: *
6: * Redistribution and use in source and binary forms, with or without
7: * modification, are permitted provided that the following conditions
8: * are met:
9: * 1. Redistributions of source code must retain the above copyright
10: * notice immediately at the beginning of the file, without modification,
11: * this list of conditions, and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20: * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26: * SUCH DAMAGE.
27: */
28: /*
29: * Encoding -- determine the character encoding of a text file.
30: *
31: * Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
32: * international characters.
33: */
34:
35: #include "file.h"
36:
37: #ifndef lint
38: FILE_RCSID("@(#)$File: encoding.c,v 1.3 2009/02/03 20:27:51 christos Exp $")
39: #endif /* lint */
40:
41: #include "magic.h"
42: #include <string.h>
43: #include <memory.h>
44: #include <stdlib.h>
45:
46:
47: private int looks_ascii(const unsigned char *, size_t, unichar *, size_t *);
48: private int looks_utf8_with_BOM(const unsigned char *, size_t, unichar *,
49: size_t *);
50: private int looks_ucs16(const unsigned char *, size_t, unichar *, size_t *);
51: private int looks_latin1(const unsigned char *, size_t, unichar *, size_t *);
52: private int looks_extended(const unsigned char *, size_t, unichar *, size_t *);
53: private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
54:
55: /*
56: * Try to determine whether text is in some character code we can
57: * identify. Each of these tests, if it succeeds, will leave
58: * the text converted into one-unichar-per-character Unicode in
59: * ubuf, and the number of characters converted in ulen.
60: */
61: protected int
62: file_encoding(struct magic_set *ms, const unsigned char *buf, size_t nbytes, unichar **ubuf, size_t *ulen, const char **code, const char **code_mime, const char **type)
63: {
64: size_t mlen;
65: int rv = 1, ucs_type;
66: unsigned char *nbuf = NULL;
67:
68: mlen = (nbytes + 1) * sizeof(nbuf[0]);
69: if ((nbuf = CAST(unsigned char *, calloc((size_t)1, mlen))) == NULL) {
70: file_oomem(ms, mlen);
71: goto done;
72: }
73: mlen = (nbytes + 1) * sizeof((*ubuf)[0]);
74: if ((*ubuf = CAST(unichar *, calloc((size_t)1, mlen))) == NULL) {
75: file_oomem(ms, mlen);
76: goto done;
77: }
78:
79: *type = "text";
80: if (looks_ascii(buf, nbytes, *ubuf, ulen)) {
81: *code = "ASCII";
82: *code_mime = "us-ascii";
83: } else if (looks_utf8_with_BOM(buf, nbytes, *ubuf, ulen) > 0) {
84: *code = "UTF-8 Unicode (with BOM)";
85: *code_mime = "utf-8";
86: } else if (file_looks_utf8(buf, nbytes, *ubuf, ulen) > 1) {
87: *code = "UTF-8 Unicode";
88: *code_mime = "utf-8";
89: } else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {
90: if (ucs_type == 1) {
91: *code = "Little-endian UTF-16 Unicode";
92: *code_mime = "utf-16le";
93: } else {
94: *code = "Big-endian UTF-16 Unicode";
95: *code_mime = "utf-16be";
96: }
97: } else if (looks_latin1(buf, nbytes, *ubuf, ulen)) {
98: *code = "ISO-8859";
99: *code_mime = "iso-8859-1";
100: } else if (looks_extended(buf, nbytes, *ubuf, ulen)) {
101: *code = "Non-ISO extended-ASCII";
102: *code_mime = "unknown-8bit";
103: } else {
104: from_ebcdic(buf, nbytes, nbuf);
105:
106: if (looks_ascii(nbuf, nbytes, *ubuf, ulen)) {
107: *code = "EBCDIC";
108: *code_mime = "ebcdic";
109: } else if (looks_latin1(nbuf, nbytes, *ubuf, ulen)) {
110: *code = "International EBCDIC";
111: *code_mime = "ebcdic";
112: } else { /* Doesn't look like text at all */
113: rv = 0;
114: *type = "binary";
115: }
116: }
117:
118: done:
119: if (nbuf)
120: free(nbuf);
121:
122: return rv;
123: }
124:
125: /*
126: * This table reflects a particular philosophy about what constitutes
127: * "text," and there is room for disagreement about it.
128: *
129: * Version 3.31 of the file command considered a file to be ASCII if
130: * each of its characters was approved by either the isascii() or
131: * isalpha() function. On most systems, this would mean that any
132: * file consisting only of characters in the range 0x00 ... 0x7F
133: * would be called ASCII text, but many systems might reasonably
134: * consider some characters outside this range to be alphabetic,
135: * so the file command would call such characters ASCII. It might
136: * have been more accurate to call this "considered textual on the
137: * local system" than "ASCII."
138: *
139: * It considered a file to be "International language text" if each
140: * of its characters was either an ASCII printing character (according
141: * to the real ASCII standard, not the above test), a character in
142: * the range 0x80 ... 0xFF, or one of the following control characters:
143: * backspace, tab, line feed, vertical tab, form feed, carriage return,
144: * escape. No attempt was made to determine the language in which files
145: * of this type were written.
146: *
147: *
148: * The table below considers a file to be ASCII if all of its characters
149: * are either ASCII printing characters (again, according to the X3.4
150: * standard, not isascii()) or any of the following controls: bell,
151: * backspace, tab, line feed, form feed, carriage return, esc, nextline.
152: *
153: * I include bell because some programs (particularly shell scripts)
154: * use it literally, even though it is rare in normal text. I exclude
155: * vertical tab because it never seems to be used in real text. I also
156: * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
157: * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
158: * character to. It might be more appropriate to include it in the 8859
159: * set instead of the ASCII set, but it's got to be included in *something*
160: * we recognize or EBCDIC files aren't going to be considered textual.
161: * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
162: * and Latin characters, so these should possibly be allowed. But they
163: * make a real mess on VT100-style displays if they're not paired properly,
164: * so we are probably better off not calling them text.
165: *
166: * A file is considered to be ISO-8859 text if its characters are all
167: * either ASCII, according to the above definition, or printing characters
168: * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
169: *
170: * Finally, a file is considered to be international text from some other
171: * character code if its characters are all either ISO-8859 (according to
172: * the above definition) or characters in the range 0x80 ... 0x9F, which
173: * ISO-8859 considers to be control characters but the IBM PC and Macintosh
174: * consider to be printing characters.
175: */
176:
177: #define F 0 /* character never appears in text */
178: #define T 1 /* character appears in plain ASCII text */
179: #define I 2 /* character appears in ISO-8859 text */
180: #define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
181:
182: private char text_chars[256] = {
183: /* BEL BS HT LF FF CR */
184: F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F, /* 0x0X */
185: /* ESC */
186: F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
187: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
188: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
189: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
190: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
191: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
192: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
193: /* NEL */
194: X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
195: X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
196: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
197: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
198: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
199: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
200: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
201: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
202: };
203:
204: private int
205: looks_ascii(const unsigned char *buf, size_t nbytes, unichar *ubuf,
206: size_t *ulen)
207: {
208: size_t i;
209:
210: *ulen = 0;
211:
212: for (i = 0; i < nbytes; i++) {
213: int t = text_chars[buf[i]];
214:
215: if (t != T)
216: return 0;
217:
218: ubuf[(*ulen)++] = buf[i];
219: }
220:
221: return 1;
222: }
223:
224: private int
225: looks_latin1(const unsigned char *buf, size_t nbytes, unichar *ubuf, size_t *ulen)
226: {
227: size_t i;
228:
229: *ulen = 0;
230:
231: for (i = 0; i < nbytes; i++) {
232: int t = text_chars[buf[i]];
233:
234: if (t != T && t != I)
235: return 0;
236:
237: ubuf[(*ulen)++] = buf[i];
238: }
239:
240: return 1;
241: }
242:
243: private int
244: looks_extended(const unsigned char *buf, size_t nbytes, unichar *ubuf,
245: size_t *ulen)
246: {
247: size_t i;
248:
249: *ulen = 0;
250:
251: for (i = 0; i < nbytes; i++) {
252: int t = text_chars[buf[i]];
253:
254: if (t != T && t != I && t != X)
255: return 0;
256:
257: ubuf[(*ulen)++] = buf[i];
258: }
259:
260: return 1;
261: }
262:
263: /*
264: * Decide whether some text looks like UTF-8. Returns:
265: *
266: * -1: invalid UTF-8
267: * 0: uses odd control characters, so doesn't look like text
268: * 1: 7-bit text
269: * 2: definitely UTF-8 text (valid high-bit set bytes)
270: *
271: * If ubuf is non-NULL on entry, text is decoded into ubuf, *ulen;
272: * ubuf must be big enough!
273: */
274: protected int
275: file_looks_utf8(const unsigned char *buf, size_t nbytes, unichar *ubuf, size_t *ulen)
276: {
277: size_t i;
278: int n;
279: unichar c;
280: int gotone = 0, ctrl = 0;
281:
282: if (ubuf)
283: *ulen = 0;
284:
285: for (i = 0; i < nbytes; i++) {
286: if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
287: /*
288: * Even if the whole file is valid UTF-8 sequences,
289: * still reject it if it uses weird control characters.
290: */
291:
292: if (text_chars[buf[i]] != T)
293: ctrl = 1;
294:
295: if (ubuf)
296: ubuf[(*ulen)++] = buf[i];
297: } else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
298: return -1;
299: } else { /* 11xxxxxx begins UTF-8 */
300: int following;
301:
302: if ((buf[i] & 0x20) == 0) { /* 110xxxxx */
303: c = buf[i] & 0x1f;
304: following = 1;
305: } else if ((buf[i] & 0x10) == 0) { /* 1110xxxx */
306: c = buf[i] & 0x0f;
307: following = 2;
308: } else if ((buf[i] & 0x08) == 0) { /* 11110xxx */
309: c = buf[i] & 0x07;
310: following = 3;
311: } else if ((buf[i] & 0x04) == 0) { /* 111110xx */
312: c = buf[i] & 0x03;
313: following = 4;
314: } else if ((buf[i] & 0x02) == 0) { /* 1111110x */
315: c = buf[i] & 0x01;
316: following = 5;
317: } else
318: return -1;
319:
320: for (n = 0; n < following; n++) {
321: i++;
322: if (i >= nbytes)
323: goto done;
324:
325: if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
326: return -1;
327:
328: c = (c << 6) + (buf[i] & 0x3f);
329: }
330:
331: if (ubuf)
332: ubuf[(*ulen)++] = c;
333: gotone = 1;
334: }
335: }
336: done:
337: return ctrl ? 0 : (gotone ? 2 : 1);
338: }
339:
340: /*
341: * Decide whether some text looks like UTF-8 with BOM. If there is no
342: * BOM, return -1; otherwise return the result of looks_utf8 on the
343: * rest of the text.
344: */
345: private int
346: looks_utf8_with_BOM(const unsigned char *buf, size_t nbytes, unichar *ubuf,
347: size_t *ulen)
348: {
349: if (nbytes > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf)
350: return file_looks_utf8(buf + 3, nbytes - 3, ubuf, ulen);
351: else
352: return -1;
353: }
354:
355: private int
356: looks_ucs16(const unsigned char *buf, size_t nbytes, unichar *ubuf,
357: size_t *ulen)
358: {
359: int bigend;
360: size_t i;
361:
362: if (nbytes < 2)
363: return 0;
364:
365: if (buf[0] == 0xff && buf[1] == 0xfe)
366: bigend = 0;
367: else if (buf[0] == 0xfe && buf[1] == 0xff)
368: bigend = 1;
369: else
370: return 0;
371:
372: *ulen = 0;
373:
374: for (i = 2; i + 1 < nbytes; i += 2) {
375: /* XXX fix to properly handle chars > 65536 */
376:
377: if (bigend)
378: ubuf[(*ulen)++] = buf[i + 1] + 256 * buf[i];
379: else
380: ubuf[(*ulen)++] = buf[i] + 256 * buf[i + 1];
381:
382: if (ubuf[*ulen - 1] == 0xfffe)
383: return 0;
384: if (ubuf[*ulen - 1] < 128 &&
385: text_chars[(size_t)ubuf[*ulen - 1]] != T)
386: return 0;
387: }
388:
389: return 1 + bigend;
390: }
391:
392: #undef F
393: #undef T
394: #undef I
395: #undef X
396:
397: /*
398: * This table maps each EBCDIC character to an (8-bit extended) ASCII
399: * character, as specified in the rationale for the dd(1) command in
400: * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
401: *
402: * Unfortunately it does not seem to correspond exactly to any of the
403: * five variants of EBCDIC documented in IBM's _Enterprise Systems
404: * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
405: * Edition, July, 1999, pp. I-1 - I-4.
406: *
407: * Fortunately, though, all versions of EBCDIC, including this one, agree
408: * on most of the printing characters that also appear in (7-bit) ASCII.
409: * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
410: *
411: * Fortunately too, there is general agreement that codes 0x00 through
412: * 0x3F represent control characters, 0x41 a nonbreaking space, and the
413: * remainder printing characters.
414: *
415: * This is sufficient to allow us to identify EBCDIC text and to distinguish
416: * between old-style and internationalized examples of text.
417: */
418:
419: private unsigned char ebcdic_to_ascii[] = {
420: 0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15,
421: 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31,
422: 128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7,
423: 144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26,
424: ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
425: '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
426: '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
427: 186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
428: 195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
429: 202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
430: 209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
431: 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
432: '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
433: '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
434: '\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
435: '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
436: };
437:
438: #ifdef notdef
439: /*
440: * The following EBCDIC-to-ASCII table may relate more closely to reality,
441: * or at least to modern reality. It comes from
442: *
443: * http://ftp.s390.ibm.com/products/oe/bpxqp9.html
444: *
445: * and maps the characters of EBCDIC code page 1047 (the code used for
446: * Unix-derived software on IBM's 390 systems) to the corresponding
447: * characters from ISO 8859-1.
448: *
449: * If this table is used instead of the above one, some of the special
450: * cases for the NEL character can be taken out of the code.
451: */
452:
453: private unsigned char ebcdic_1047_to_8859[] = {
454: 0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
455: 0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
456: 0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
457: 0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
458: 0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
459: 0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
460: 0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
461: 0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
462: 0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
463: 0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
464: 0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
465: 0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
466: 0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
467: 0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
468: 0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
469: 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
470: };
471: #endif
472:
473: /*
474: * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
475: */
476: private void
477: from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
478: {
479: size_t i;
480:
481: for (i = 0; i < nbytes; i++) {
482: out[i] = ebcdic_to_ascii[buf[i]];
483: }
484: }
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to encoding.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / php / ext / fileinfo / libmagic