Annotation of embedaddon/libxml2/genUnicode.py, revision 1.1.1.1
1.1 misho 1: #!/usr/bin/python -u
2: #
3: # Original script modified in November 2003 to take advantage of
4: # the character-validation range routines, and updated to the
5: # current Unicode information (Version 4.0.1)
6: #
7: # NOTE: there is an 'alias' facility for blocks which are not present in
8: # the current release, but are needed for ABI compatibility. This
9: # must be accomplished MANUALLY! Please see the comments below under
10: # 'blockAliases'
11: #
12: import sys
13: import string
14: import time
15:
16: webpage = "http://www.unicode.org/Public/4.0-Update1/UCD-4.0.1.html"
17: sources = "Blocks-4.0.1.txt UnicodeData-4.0.1.txt"
18:
19: #
20: # blockAliases is a small hack - it is used for mapping block names which
21: # were were used in the 3.1 release, but are missing or changed in the current
22: # release. The format is "OldBlockName:NewBlockName1[,NewBlockName2[,...]]"
23: blockAliases = []
24: blockAliases.append("CombiningMarksforSymbols:CombiningDiacriticalMarksforSymbols")
25: blockAliases.append("Greek:GreekandCoptic")
26: blockAliases.append("PrivateUse:PrivateUseArea,SupplementaryPrivateUseArea-A," +
27: "SupplementaryPrivateUseArea-B")
28:
29: # minTableSize gives the minimum number of ranges which must be present
30: # before a range table is produced. If there are less than this
31: # number, inline comparisons are generated
32: minTableSize = 8
33:
34: (blockfile, catfile) = string.split(sources)
35:
36:
37: #
38: # Now process the "blocks" file, reducing it to a dictionary
39: # indexed by blockname, containing a tuple with the applicable
40: # block range
41: #
42: BlockNames = {}
43: try:
44: blocks = open(blockfile, "r")
45: except:
46: print "Missing %s, aborting ..." % blockfile
47: sys.exit(1)
48:
49: for line in blocks.readlines():
50: if line[0] == '#':
51: continue
52: line = string.strip(line)
53: if line == '':
54: continue
55: try:
56: fields = string.split(line, ';')
57: range = string.strip(fields[0])
58: (start, end) = string.split(range, "..")
59: name = string.strip(fields[1])
60: name = string.replace(name, ' ', '')
61: except:
62: print "Failed to process line: %s" % (line)
63: continue
64: start = "0x" + start
65: end = "0x" + end
66: try:
67: BlockNames[name].append((start, end))
68: except:
69: BlockNames[name] = [(start, end)]
70: blocks.close()
71: print "Parsed %d blocks descriptions" % (len(BlockNames.keys()))
72:
73: for block in blockAliases:
74: alias = string.split(block,':')
75: alist = string.split(alias[1],',')
76: for comp in alist:
77: if BlockNames.has_key(comp):
78: if alias[0] not in BlockNames:
79: BlockNames[alias[0]] = []
80: for r in BlockNames[comp]:
81: BlockNames[alias[0]].append(r)
82: else:
83: print "Alias %s: %s not in Blocks" % (alias[0], comp)
84: continue
85:
86: #
87: # Next process the Categories file. This is more complex, since
88: # the file is in code sequence, and we need to invert it. We use
89: # a dictionary with index category-name, with each entry containing
90: # all the ranges (codepoints) of that category. Note that category
91: # names comprise two parts - the general category, and the "subclass"
92: # within that category. Therefore, both "general category" (which is
93: # the first character of the 2-character category-name) and the full
94: # (2-character) name are entered into this dictionary.
95: #
96: try:
97: data = open(catfile, "r")
98: except:
99: print "Missing %s, aborting ..." % catfile
100: sys.exit(1)
101:
102: nbchar = 0;
103: Categories = {}
104: for line in data.readlines():
105: if line[0] == '#':
106: continue
107: line = string.strip(line)
108: if line == '':
109: continue
110: try:
111: fields = string.split(line, ';')
112: point = string.strip(fields[0])
113: value = 0
114: while point != '':
115: value = value * 16
116: if point[0] >= '0' and point[0] <= '9':
117: value = value + ord(point[0]) - ord('0')
118: elif point[0] >= 'A' and point[0] <= 'F':
119: value = value + 10 + ord(point[0]) - ord('A')
120: elif point[0] >= 'a' and point[0] <= 'f':
121: value = value + 10 + ord(point[0]) - ord('a')
122: point = point[1:]
123: name = fields[2]
124: except:
125: print "Failed to process line: %s" % (line)
126: continue
127:
128: nbchar = nbchar + 1
129: # update entry for "full name"
130: try:
131: Categories[name].append(value)
132: except:
133: try:
134: Categories[name] = [value]
135: except:
136: print "Failed to process line: %s" % (line)
137: # update "general category" name
138: try:
139: Categories[name[0]].append(value)
140: except:
141: try:
142: Categories[name[0]] = [value]
143: except:
144: print "Failed to process line: %s" % (line)
145:
146: blocks.close()
147: print "Parsed %d char generating %d categories" % (nbchar, len(Categories.keys()))
148:
149: #
150: # The data is now all read. Time to process it into a more useful form.
151: #
152: # reduce the number list into ranges
153: for cat in Categories.keys():
154: list = Categories[cat]
155: start = -1
156: prev = -1
157: end = -1
158: ranges = []
159: for val in list:
160: if start == -1:
161: start = val
162: prev = val
163: continue
164: elif val == prev + 1:
165: prev = val
166: continue
167: elif prev == start:
168: ranges.append((prev, prev))
169: start = val
170: prev = val
171: continue
172: else:
173: ranges.append((start, prev))
174: start = val
175: prev = val
176: continue
177: if prev == start:
178: ranges.append((prev, prev))
179: else:
180: ranges.append((start, prev))
181: Categories[cat] = ranges
182:
183: #
184: # Assure all data is in alphabetic order, since we will be doing binary
185: # searches on the tables.
186: #
187: bkeys = BlockNames.keys()
188: bkeys.sort()
189:
190: ckeys = Categories.keys()
191: ckeys.sort()
192:
193: #
194: # Generate the resulting files
195: #
196: try:
197: header = open("include/libxml/xmlunicode.h", "w")
198: except:
199: print "Failed to open include/libxml/xmlunicode.h"
200: sys.exit(1)
201:
202: try:
203: output = open("xmlunicode.c", "w")
204: except:
205: print "Failed to open xmlunicode.c"
206: sys.exit(1)
207:
208: date = time.asctime(time.localtime(time.time()))
209:
210: header.write(
211: """/*
212: * Summary: Unicode character APIs
213: * Description: API for the Unicode character APIs
214: *
215: * This file is automatically generated from the
216: * UCS description files of the Unicode Character Database
217: * %s
218: * using the genUnicode.py Python script.
219: *
220: * Generation date: %s
221: * Sources: %s
222: * Author: Daniel Veillard
223: */
224:
225: #ifndef __XML_UNICODE_H__
226: #define __XML_UNICODE_H__
227:
228: #include <libxml/xmlversion.h>
229:
230: #ifdef LIBXML_UNICODE_ENABLED
231:
232: #ifdef __cplusplus
233: extern "C" {
234: #endif
235:
236: """ % (webpage, date, sources));
237:
238: output.write(
239: """/*
240: * xmlunicode.c: this module implements the Unicode character APIs
241: *
242: * This file is automatically generated from the
243: * UCS description files of the Unicode Character Database
244: * %s
245: * using the genUnicode.py Python script.
246: *
247: * Generation date: %s
248: * Sources: %s
249: * Daniel Veillard <veillard@redhat.com>
250: */
251:
252: #define IN_LIBXML
253: #include "libxml.h"
254:
255: #ifdef LIBXML_UNICODE_ENABLED
256:
257: #include <string.h>
258: #include <libxml/xmlversion.h>
259: #include <libxml/xmlunicode.h>
260: #include <libxml/chvalid.h>
261:
262: typedef int (xmlIntFunc)(int); /* just to keep one's mind untwisted */
263:
264: typedef struct {
265: const char *rangename;
266: xmlIntFunc *func;
267: } xmlUnicodeRange;
268:
269: typedef struct {
270: xmlUnicodeRange *table;
271: int numentries;
272: } xmlUnicodeNameTable;
273:
274:
275: static xmlIntFunc *xmlUnicodeLookup(xmlUnicodeNameTable *tptr, const char *tname);
276:
277: static xmlUnicodeRange xmlUnicodeBlocks[] = {
278: """ % (webpage, date, sources));
279:
280: flag = 0
281: for block in bkeys:
282: name = string.replace(block, '-', '')
283: if flag:
284: output.write(',\n')
285: else:
286: flag = 1
287: output.write(' {"%s", xmlUCSIs%s}' % (block, name))
288: output.write('};\n\n')
289:
290: output.write('static xmlUnicodeRange xmlUnicodeCats[] = {\n')
291: flag = 0;
292: for name in ckeys:
293: if flag:
294: output.write(',\n')
295: else:
296: flag = 1
297: output.write(' {"%s", xmlUCSIsCat%s}' % (name, name))
298: output.write('};\n\n')
299:
300: #
301: # For any categories with more than minTableSize ranges we generate
302: # a range table suitable for xmlCharInRange
303: #
304: for name in ckeys:
305: if len(Categories[name]) > minTableSize:
306: numshort = 0
307: numlong = 0
308: ranges = Categories[name]
309: sptr = "NULL"
310: lptr = "NULL"
311: for range in ranges:
312: (low, high) = range
313: if high < 0x10000:
314: if numshort == 0:
315: pline = "static const xmlChSRange xml%sS[] = {" % name
316: sptr = "xml%sS" % name
317: else:
318: pline += ", "
319: numshort += 1
320: else:
321: if numlong == 0:
322: if numshort > 0:
323: output.write(pline + " };\n")
324: pline = "static const xmlChLRange xml%sL[] = {" % name
325: lptr = "xml%sL" % name
326: else:
327: pline += ", "
328: numlong += 1
329: if len(pline) > 60:
330: output.write(pline + "\n")
331: pline = " "
332: pline += "{%s, %s}" % (hex(low), hex(high))
333: output.write(pline + " };\nstatic xmlChRangeGroup xml%sG = {%s,%s,%s,%s};\n\n"
334: % (name, numshort, numlong, sptr, lptr))
335:
336:
337: output.write(
338: """static xmlUnicodeNameTable xmlUnicodeBlockTbl = {xmlUnicodeBlocks, %s};
339: static xmlUnicodeNameTable xmlUnicodeCatTbl = {xmlUnicodeCats, %s};
340:
341: /**
342: * xmlUnicodeLookup:
343: * @tptr: pointer to the name table
344: * @name: name to be found
345: *
346: * binary table lookup for user-supplied name
347: *
348: * Returns pointer to range function if found, otherwise NULL
349: */
350: static xmlIntFunc
351: *xmlUnicodeLookup(xmlUnicodeNameTable *tptr, const char *tname) {
352: int low, high, mid, cmp;
353: xmlUnicodeRange *sptr;
354:
355: if ((tptr == NULL) || (tname == NULL)) return(NULL);
356:
357: low = 0;
358: high = tptr->numentries - 1;
359: sptr = tptr->table;
360: while (low <= high) {
361: mid = (low + high) / 2;
362: if ((cmp=strcmp(tname, sptr[mid].rangename)) == 0)
363: return (sptr[mid].func);
364: if (cmp < 0)
365: high = mid - 1;
366: else
367: low = mid + 1;
368: }
369: return (NULL);
370: }
371:
372: """ % (len(BlockNames), len(Categories)) )
373:
374: for block in bkeys:
375: name = string.replace(block, '-', '')
376: header.write("XMLPUBFUN int XMLCALL xmlUCSIs%s\t(int code);\n" % name)
377: output.write("/**\n * xmlUCSIs%s:\n * @code: UCS code point\n" % (name))
378: output.write(" *\n * Check whether the character is part of %s UCS Block\n"%
379: (block))
380: output.write(" *\n * Returns 1 if true 0 otherwise\n */\n");
381: output.write("int\nxmlUCSIs%s(int code) {\n return(" % name)
382: flag = 0
383: for (start, end) in BlockNames[block]:
384: if flag:
385: output.write(" ||\n ")
386: else:
387: flag = 1
388: output.write("((code >= %s) && (code <= %s))" % (start, end))
389: output.write(");\n}\n\n")
390:
391: header.write("\nXMLPUBFUN int XMLCALL xmlUCSIsBlock\t(int code, const char *block);\n\n")
392: output.write(
393: """/**
394: * xmlUCSIsBlock:
395: * @code: UCS code point
396: * @block: UCS block name
397: *
398: * Check whether the character is part of the UCS Block
399: *
400: * Returns 1 if true, 0 if false and -1 on unknown block
401: */
402: int
403: xmlUCSIsBlock(int code, const char *block) {
404: xmlIntFunc *func;
405:
406: func = xmlUnicodeLookup(&xmlUnicodeBlockTbl, block);
407: if (func == NULL)
408: return (-1);
409: return (func(code));
410: }
411:
412: """)
413:
414: for name in ckeys:
415: ranges = Categories[name]
416: header.write("XMLPUBFUN int XMLCALL xmlUCSIsCat%s\t(int code);\n" % name)
417: output.write("/**\n * xmlUCSIsCat%s:\n * @code: UCS code point\n" % (name))
418: output.write(" *\n * Check whether the character is part of %s UCS Category\n"%
419: (name))
420: output.write(" *\n * Returns 1 if true 0 otherwise\n */\n");
421: output.write("int\nxmlUCSIsCat%s(int code) {\n" % name)
422: if len(Categories[name]) > minTableSize:
423: output.write(" return(xmlCharInRange((unsigned int)code, &xml%sG)"
424: % name)
425: else:
426: start = 1
427: for range in ranges:
428: (begin, end) = range;
429: if start:
430: output.write(" return(");
431: start = 0
432: else:
433: output.write(" ||\n ");
434: if (begin == end):
435: output.write("(code == %s)" % (hex(begin)))
436: else:
437: output.write("((code >= %s) && (code <= %s))" % (
438: hex(begin), hex(end)))
439: output.write(");\n}\n\n")
440:
441: header.write("\nXMLPUBFUN int XMLCALL xmlUCSIsCat\t(int code, const char *cat);\n")
442: output.write(
443: """/**
444: * xmlUCSIsCat:
445: * @code: UCS code point
446: * @cat: UCS Category name
447: *
448: * Check whether the character is part of the UCS Category
449: *
450: * Returns 1 if true, 0 if false and -1 on unknown category
451: */
452: int
453: xmlUCSIsCat(int code, const char *cat) {
454: xmlIntFunc *func;
455:
456: func = xmlUnicodeLookup(&xmlUnicodeCatTbl, cat);
457: if (func == NULL)
458: return (-1);
459: return (func(code));
460: }
461:
462: #define bottom_xmlunicode
463: #include "elfgcchack.h"
464: #endif /* LIBXML_UNICODE_ENABLED */
465: """)
466:
467: header.write("""
468: #ifdef __cplusplus
469: }
470: #endif
471:
472: #endif /* LIBXML_UNICODE_ENABLED */
473:
474: #endif /* __XML_UNICODE_H__ */
475: """);
476:
477: header.close()
478: output.close()
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to genUnicode.py CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / libxml2