1: #!/usr/bin/env python
2:
3: #
4: # This is the MS subset of the W3C test suite for XML Schemas.
5: # This file is generated from the MS W3c test suite description file.
6: #
7:
8: import sys, os
9: import exceptions, optparse
10: import libxml2
11:
12: opa = optparse.OptionParser()
13:
14: opa.add_option("-b", "--base", action="store", type="string", dest="baseDir",
15: default="",
16: help="""The base directory; i.e. the parent folder of the
17: "nisttest", "suntest" and "msxsdtest" directories.""")
18:
19: opa.add_option("-o", "--out", action="store", type="string", dest="logFile",
20: default="test.log",
21: help="The filepath of the log file to be created")
22:
23: opa.add_option("--log", action="store_true", dest="enableLog",
24: default=False,
25: help="Create the log file")
26:
27: opa.add_option("--no-test-out", action="store_true", dest="disableTestStdOut",
28: default=False,
29: help="Don't output test results")
30:
31: opa.add_option("-s", "--silent", action="store_true", dest="silent", default=False,
32: help="Disables display of all tests")
33:
34: opa.add_option("-v", "--verbose", action="store_true", dest="verbose",
35: default=False,
36: help="Displays all tests (only if --silent is not set)")
37:
38: opa.add_option("-x", "--max", type="int", dest="maxTestCount",
39: default="-1",
40: help="The maximum number of tests to be run")
41:
42: opa.add_option("-t", "--test", type="string", dest="singleTest",
43: default=None,
44: help="Runs the specified test only")
45:
46: opa.add_option("--tsw", "--test-starts-with", type="string", dest="testStartsWith",
47: default=None,
48: help="Runs the specified test(s), starting with the given string")
49:
50: opa.add_option("--rieo", "--report-internal-errors-only", action="store_true",
51: dest="reportInternalErrOnly", default=False,
52: help="Display erroneous tests of type 'internal' only")
53:
54: opa.add_option("--rueo", "--report-unimplemented-errors-only", action="store_true",
55: dest="reportUnimplErrOnly", default=False,
56: help="Display erroneous tests of type 'unimplemented' only")
57:
58: opa.add_option("--rmleo", "--report-mem-leak-errors-only", action="store_true",
59: dest="reportMemLeakErrOnly", default=False,
60: help="Display erroneous tests of type 'memory leak' only")
61:
62: opa.add_option("-c", "--combines", type="string", dest="combines",
63: default=None,
64: help="Combines to be run (all if omitted)")
65:
66: opa.add_option("--csw", "--csw", type="string", dest="combineStartsWith",
67: default=None,
68: help="Combines to be run (all if omitted)")
69:
70: opa.add_option("--rc", "--report-combines", action="store_true",
71: dest="reportCombines", default=False,
72: help="Display combine reports")
73:
74: opa.add_option("--rec", "--report-err-combines", action="store_true",
75: dest="reportErrCombines", default=False,
76: help="Display erroneous combine reports only")
77:
78: opa.add_option("--debug", action="store_true",
79: dest="debugEnabled", default=False,
80: help="Displays debug messages")
81:
82: opa.add_option("--info", action="store_true",
83: dest="info", default=False,
84: help="Displays info on the suite only. Does not run any test.")
85: opa.add_option("--sax", action="store_true",
86: dest="validationSAX", default=False,
87: help="Use SAX2-driven validation.")
88: opa.add_option("--tn", action="store_true",
89: dest="displayTestName", default=False,
90: help="Display the test name in every case.")
91:
92: (options, args) = opa.parse_args()
93:
94: if options.combines is not None:
95: options.combines = options.combines.split()
96:
97: ################################################
98: # The vars below are not intended to be changed.
99: #
100:
101: msgSchemaNotValidButShould = "The schema should be valid."
102: msgSchemaValidButShouldNot = "The schema should be invalid."
103: msgInstanceNotValidButShould = "The instance should be valid."
104: msgInstanceValidButShouldNot = "The instance should be invalid."
105: vendorNIST = "NIST"
106: vendorNIST_2 = "NIST-2"
107: vendorSUN = "SUN"
108: vendorMS = "MS"
109:
110: ###################
111: # Helper functions.
112: #
113: vendor = None
114:
115: def handleError(test, msg):
116: global options
117: if not options.silent:
118: test.addLibLog("'%s' LIB: %s" % (test.name, msg))
119: if msg.find("Unimplemented") > -1:
120: test.failUnimplemented()
121: elif msg.find("Internal") > -1:
122: test.failInternal()
123:
124:
125: def fixFileNames(fileName):
126: if (fileName is None) or (fileName == ""):
127: return ""
128: dirs = fileName.split("/")
129: if dirs[1] != "Tests":
130: fileName = os.path.join(".", "Tests")
131: for dir in dirs[1:]:
132: fileName = os.path.join(fileName, dir)
133: return fileName
134:
135: class XSTCTestGroup:
136: def __init__(self, name, schemaFileName, descr):
137: global vendor, vendorNIST_2
138: self.name = name
139: self.descr = descr
140: self.mainSchema = True
141: self.schemaFileName = fixFileNames(schemaFileName)
142: self.schemaParsed = False
143: self.schemaTried = False
144:
145: def setSchema(self, schemaFileName, parsed):
146: if not self.mainSchema:
147: return
148: self.mainSchema = False
149: self.schemaParsed = parsed
150: self.schemaTried = True
151:
152: class XSTCTestCase:
153:
154: # <!-- groupName, Name, Accepted, File, Val, Descr
155: def __init__(self, isSchema, groupName, name, accepted, file, val, descr):
156: global options
157: #
158: # Constructor.
159: #
160: self.testRunner = None
161: self.isSchema = isSchema
162: self.groupName = groupName
163: self.name = name
164: self.accepted = accepted
165: self.fileName = fixFileNames(file)
166: self.val = val
167: self.descr = descr
168: self.failed = False
169: self.combineName = None
170:
171: self.log = []
172: self.libLog = []
173: self.initialMemUsed = 0
174: self.memLeak = 0
175: self.excepted = False
176: self.bad = False
177: self.unimplemented = False
178: self.internalErr = False
179: self.noSchemaErr = False
180: self.failed = False
181: #
182: # Init the log.
183: #
184: if not options.silent:
185: if self.descr is not None:
186: self.log.append("'%s' descr: %s\n" % (self.name, self.descr))
187: self.log.append("'%s' exp validity: %d\n" % (self.name, self.val))
188:
189: def initTest(self, runner):
190: global vendorNIST, vendorSUN, vendorMS, vendorNIST_2, options, vendor
191: #
192: # Get the test-group.
193: #
194: self.runner = runner
195: self.group = runner.getGroup(self.groupName)
196: if vendor == vendorMS or vendor == vendorSUN:
197: #
198: # Use the last given directory for the combine name.
199: #
200: dirs = self.fileName.split("/")
201: self.combineName = dirs[len(dirs) -2]
202: elif vendor == vendorNIST:
203: #
204: # NIST files are named in the following form:
205: # "NISTSchema-short-pattern-1.xsd"
206: #
207: tokens = self.name.split("-")
208: self.combineName = tokens[1]
209: elif vendor == vendorNIST_2:
210: #
211: # Group-names have the form: "atomic-normalizedString-length-1"
212: #
213: tokens = self.groupName.split("-")
214: self.combineName = "%s-%s" % (tokens[0], tokens[1])
215: else:
216: self.combineName = "unkown"
217: raise Exception("Could not compute the combine name of a test.")
218: if (not options.silent) and (self.group.descr is not None):
219: self.log.append("'%s' group-descr: %s\n" % (self.name, self.group.descr))
220:
221:
222: def addLibLog(self, msg):
223: """This one is intended to be used by the error handler
224: function"""
225: global options
226: if not options.silent:
227: self.libLog.append(msg)
228:
229: def fail(self, msg):
230: global options
231: self.failed = True
232: if not options.silent:
233: self.log.append("'%s' ( FAILED: %s\n" % (self.name, msg))
234:
235: def failNoSchema(self):
236: global options
237: self.failed = True
238: self.noSchemaErr = True
239: if not options.silent:
240: self.log.append("'%s' X NO-SCHEMA\n" % (self.name))
241:
242: def failInternal(self):
243: global options
244: self.failed = True
245: self.internalErr = True
246: if not options.silent:
247: self.log.append("'%s' * INTERNAL\n" % self.name)
248:
249: def failUnimplemented(self):
250: global options
251: self.failed = True
252: self.unimplemented = True
253: if not options.silent:
254: self.log.append("'%s' ? UNIMPLEMENTED\n" % self.name)
255:
256: def failCritical(self, msg):
257: global options
258: self.failed = True
259: self.bad = True
260: if not options.silent:
261: self.log.append("'%s' ! BAD: %s\n" % (self.name, msg))
262:
263: def failExcept(self, e):
264: global options
265: self.failed = True
266: self.excepted = True
267: if not options.silent:
268: self.log.append("'%s' # EXCEPTION: %s\n" % (self.name, e.__str__()))
269:
270: def setUp(self):
271: #
272: # Set up Libxml2.
273: #
274: self.initialMemUsed = libxml2.debugMemory(1)
275: libxml2.initParser()
276: libxml2.lineNumbersDefault(1)
277: libxml2.registerErrorHandler(handleError, self)
278:
279: def tearDown(self):
280: libxml2.schemaCleanupTypes()
281: libxml2.cleanupParser()
282: self.memLeak = libxml2.debugMemory(1) - self.initialMemUsed
283:
284: def isIOError(self, file, docType):
285: err = None
286: try:
287: err = libxml2.lastError()
288: except:
289: # Suppress exceptions.
290: pass
291: if (err is None):
292: return False
293: if err.domain() == libxml2.XML_FROM_IO:
294: self.failCritical("failed to access the %s resource '%s'\n" % (docType, file))
295:
296: def debugMsg(self, msg):
297: global options
298: if options.debugEnabled:
299: sys.stdout.write("'%s' DEBUG: %s\n" % (self.name, msg))
300:
301: def finalize(self):
302: global options
303: """Adds additional info to the log."""
304: #
305: # Add libxml2 messages.
306: #
307: if not options.silent:
308: self.log.extend(self.libLog)
309: #
310: # Add memory leaks.
311: #
312: if self.memLeak != 0:
313: self.log.append("%s + memory leak: %d bytes\n" % (self.name, self.memLeak))
314:
315: def run(self):
316: """Runs a test."""
317: global options
318:
319: ##filePath = os.path.join(options.baseDir, self.fileName)
320: # filePath = "%s/%s/%s/%s" % (options.baseDir, self.test_Folder, self.schema_Folder, self.schema_File)
321: if options.displayTestName:
322: sys.stdout.write("'%s'\n" % self.name)
323: try:
324: self.validate()
325: except (Exception, libxml2.parserError, libxml2.treeError), e:
326: self.failExcept(e)
327:
328: def parseSchema(fileName):
329: schema = None
330: ctxt = libxml2.schemaNewParserCtxt(fileName)
331: try:
332: try:
333: schema = ctxt.schemaParse()
334: except:
335: pass
336: finally:
337: del ctxt
338: return schema
339:
340:
341: class XSTCSchemaTest(XSTCTestCase):
342:
343: def __init__(self, groupName, name, accepted, file, val, descr):
344: XSTCTestCase.__init__(self, 1, groupName, name, accepted, file, val, descr)
345:
346: def validate(self):
347: global msgSchemaNotValidButShould, msgSchemaValidButShouldNot
348: schema = None
349: filePath = self.fileName
350: # os.path.join(options.baseDir, self.fileName)
351: valid = 0
352: try:
353: #
354: # Parse the schema.
355: #
356: self.debugMsg("loading schema: %s" % filePath)
357: schema = parseSchema(filePath)
358: self.debugMsg("after loading schema")
359: if schema is None:
360: self.debugMsg("schema is None")
361: self.debugMsg("checking for IO errors...")
362: if self.isIOError(file, "schema"):
363: return
364: self.debugMsg("checking schema result")
365: if (schema is None and self.val) or (schema is not None and self.val == 0):
366: self.debugMsg("schema result is BAD")
367: if (schema == None):
368: self.fail(msgSchemaNotValidButShould)
369: else:
370: self.fail(msgSchemaValidButShouldNot)
371: else:
372: self.debugMsg("schema result is OK")
373: finally:
374: self.group.setSchema(self.fileName, schema is not None)
375: del schema
376:
377: class XSTCInstanceTest(XSTCTestCase):
378:
379: def __init__(self, groupName, name, accepted, file, val, descr):
380: XSTCTestCase.__init__(self, 0, groupName, name, accepted, file, val, descr)
381:
382: def validate(self):
383: instance = None
384: schema = None
385: filePath = self.fileName
386: # os.path.join(options.baseDir, self.fileName)
387:
388: if not self.group.schemaParsed and self.group.schemaTried:
389: self.failNoSchema()
390: return
391:
392: self.debugMsg("loading instance: %s" % filePath)
393: parserCtxt = libxml2.newParserCtxt()
394: if (parserCtxt is None):
395: # TODO: Is this one necessary, or will an exception
396: # be already raised?
397: raise Exception("Could not create the instance parser context.")
398: if not options.validationSAX:
399: try:
400: try:
401: instance = parserCtxt.ctxtReadFile(filePath, None, libxml2.XML_PARSE_NOWARNING)
402: except:
403: # Suppress exceptions.
404: pass
405: finally:
406: del parserCtxt
407: self.debugMsg("after loading instance")
408: if instance is None:
409: self.debugMsg("instance is None")
410: self.failCritical("Failed to parse the instance for unknown reasons.")
411: return
412: try:
413: #
414: # Validate the instance.
415: #
416: self.debugMsg("loading schema: %s" % self.group.schemaFileName)
417: schema = parseSchema(self.group.schemaFileName)
418: try:
419: validationCtxt = schema.schemaNewValidCtxt()
420: #validationCtxt = libxml2.schemaNewValidCtxt(None)
421: if (validationCtxt is None):
422: self.failCritical("Could not create the validation context.")
423: return
424: try:
425: self.debugMsg("validating instance")
426: if options.validationSAX:
427: instance_Err = validationCtxt.schemaValidateFile(filePath, 0)
428: else:
429: instance_Err = validationCtxt.schemaValidateDoc(instance)
430: self.debugMsg("after instance validation")
431: self.debugMsg("instance-err: %d" % instance_Err)
432: if (instance_Err != 0 and self.val == 1) or (instance_Err == 0 and self.val == 0):
433: self.debugMsg("instance result is BAD")
434: if (instance_Err != 0):
435: self.fail(msgInstanceNotValidButShould)
436: else:
437: self.fail(msgInstanceValidButShouldNot)
438:
439: else:
440: self.debugMsg("instance result is OK")
441: finally:
442: del validationCtxt
443: finally:
444: del schema
445: finally:
446: if instance is not None:
447: instance.freeDoc()
448:
449:
450: ####################
451: # Test runner class.
452: #
453:
454: class XSTCTestRunner:
455:
456: CNT_TOTAL = 0
457: CNT_RAN = 1
458: CNT_SUCCEEDED = 2
459: CNT_FAILED = 3
460: CNT_UNIMPLEMENTED = 4
461: CNT_INTERNAL = 5
462: CNT_BAD = 6
463: CNT_EXCEPTED = 7
464: CNT_MEMLEAK = 8
465: CNT_NOSCHEMA = 9
466: CNT_NOTACCEPTED = 10
467: CNT_SCHEMA_TEST = 11
468:
469: def __init__(self):
470: self.logFile = None
471: self.counters = self.createCounters()
472: self.testList = []
473: self.combinesRan = {}
474: self.groups = {}
475: self.curGroup = None
476:
477: def createCounters(self):
478: counters = {self.CNT_TOTAL:0, self.CNT_RAN:0, self.CNT_SUCCEEDED:0,
479: self.CNT_FAILED:0, self.CNT_UNIMPLEMENTED:0, self.CNT_INTERNAL:0, self.CNT_BAD:0,
480: self.CNT_EXCEPTED:0, self.CNT_MEMLEAK:0, self.CNT_NOSCHEMA:0, self.CNT_NOTACCEPTED:0,
481: self.CNT_SCHEMA_TEST:0}
482:
483: return counters
484:
485: def addTest(self, test):
486: self.testList.append(test)
487: test.initTest(self)
488:
489: def getGroup(self, groupName):
490: return self.groups[groupName]
491:
492: def addGroup(self, group):
493: self.groups[group.name] = group
494:
495: def updateCounters(self, test, counters):
496: if test.memLeak != 0:
497: counters[self.CNT_MEMLEAK] += 1
498: if not test.failed:
499: counters[self.CNT_SUCCEEDED] +=1
500: if test.failed:
501: counters[self.CNT_FAILED] += 1
502: if test.bad:
503: counters[self.CNT_BAD] += 1
504: if test.unimplemented:
505: counters[self.CNT_UNIMPLEMENTED] += 1
506: if test.internalErr:
507: counters[self.CNT_INTERNAL] += 1
508: if test.noSchemaErr:
509: counters[self.CNT_NOSCHEMA] += 1
510: if test.excepted:
511: counters[self.CNT_EXCEPTED] += 1
512: if not test.accepted:
513: counters[self.CNT_NOTACCEPTED] += 1
514: if test.isSchema:
515: counters[self.CNT_SCHEMA_TEST] += 1
516: return counters
517:
518: def displayResults(self, out, all, combName, counters):
519: out.write("\n")
520: if all:
521: if options.combines is not None:
522: out.write("combine(s): %s\n" % str(options.combines))
523: elif combName is not None:
524: out.write("combine : %s\n" % combName)
525: out.write(" total : %d\n" % counters[self.CNT_TOTAL])
526: if all or options.combines is not None:
527: out.write(" ran : %d\n" % counters[self.CNT_RAN])
528: out.write(" (schemata) : %d\n" % counters[self.CNT_SCHEMA_TEST])
529: # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED])
530: out.write(" not accepted : %d\n" % counters[self.CNT_NOTACCEPTED])
531: if counters[self.CNT_FAILED] > 0:
532: out.write(" failed : %d\n" % counters[self.CNT_FAILED])
533: out.write(" -> internal : %d\n" % counters[self.CNT_INTERNAL])
534: out.write(" -> unimpl. : %d\n" % counters[self.CNT_UNIMPLEMENTED])
535: out.write(" -> skip-invalid-schema : %d\n" % counters[self.CNT_NOSCHEMA])
536: out.write(" -> bad : %d\n" % counters[self.CNT_BAD])
537: out.write(" -> exceptions : %d\n" % counters[self.CNT_EXCEPTED])
538: out.write(" memory leaks : %d\n" % counters[self.CNT_MEMLEAK])
539:
540: def displayShortResults(self, out, all, combName, counters):
541: out.write("Ran %d of %d tests (%d schemata):" % (counters[self.CNT_RAN],
542: counters[self.CNT_TOTAL], counters[self.CNT_SCHEMA_TEST]))
543: # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED])
544: if counters[self.CNT_NOTACCEPTED] > 0:
545: out.write(" %d not accepted" % (counters[self.CNT_NOTACCEPTED]))
546: if counters[self.CNT_FAILED] > 0 or counters[self.CNT_MEMLEAK] > 0:
547: if counters[self.CNT_FAILED] > 0:
548: out.write(" %d failed" % (counters[self.CNT_FAILED]))
549: out.write(" (")
550: if counters[self.CNT_INTERNAL] > 0:
551: out.write(" %d internal" % (counters[self.CNT_INTERNAL]))
552: if counters[self.CNT_UNIMPLEMENTED] > 0:
553: out.write(" %d unimplemented" % (counters[self.CNT_UNIMPLEMENTED]))
554: if counters[self.CNT_NOSCHEMA] > 0:
555: out.write(" %d skip-invalid-schema" % (counters[self.CNT_NOSCHEMA]))
556: if counters[self.CNT_BAD] > 0:
557: out.write(" %d bad" % (counters[self.CNT_BAD]))
558: if counters[self.CNT_EXCEPTED] > 0:
559: out.write(" %d exception" % (counters[self.CNT_EXCEPTED]))
560: out.write(" )")
561: if counters[self.CNT_MEMLEAK] > 0:
562: out.write(" %d leaks" % (counters[self.CNT_MEMLEAK]))
563: out.write("\n")
564: else:
565: out.write(" all passed\n")
566:
567: def reportCombine(self, combName):
568: global options
569:
570: counters = self.createCounters()
571: #
572: # Compute evaluation counters.
573: #
574: for test in self.combinesRan[combName]:
575: counters[self.CNT_TOTAL] += 1
576: counters[self.CNT_RAN] += 1
577: counters = self.updateCounters(test, counters)
578: if options.reportErrCombines and (counters[self.CNT_FAILED] == 0) and (counters[self.CNT_MEMLEAK] == 0):
579: pass
580: else:
581: if options.enableLog:
582: self.displayResults(self.logFile, False, combName, counters)
583: self.displayResults(sys.stdout, False, combName, counters)
584:
585: def displayTestLog(self, test):
586: sys.stdout.writelines(test.log)
587: sys.stdout.write("~~~~~~~~~~\n")
588:
589: def reportTest(self, test):
590: global options
591:
592: error = test.failed or test.memLeak != 0
593: #
594: # Only erroneous tests will be written to the log,
595: # except @verbose is switched on.
596: #
597: if options.enableLog and (options.verbose or error):
598: self.logFile.writelines(test.log)
599: self.logFile.write("~~~~~~~~~~\n")
600: #
601: # if not @silent, only erroneous tests will be
602: # written to stdout, except @verbose is switched on.
603: #
604: if not options.silent:
605: if options.reportInternalErrOnly and test.internalErr:
606: self.displayTestLog(test)
607: if options.reportMemLeakErrOnly and test.memLeak != 0:
608: self.displayTestLog(test)
609: if options.reportUnimplErrOnly and test.unimplemented:
610: self.displayTestLog(test)
611: if (options.verbose or error) and (not options.reportInternalErrOnly) and (not options.reportMemLeakErrOnly) and (not options.reportUnimplErrOnly):
612: self.displayTestLog(test)
613:
614:
615: def addToCombines(self, test):
616: found = False
617: if self.combinesRan.has_key(test.combineName):
618: self.combinesRan[test.combineName].append(test)
619: else:
620: self.combinesRan[test.combineName] = [test]
621:
622: def run(self):
623:
624: global options
625:
626: if options.info:
627: for test in self.testList:
628: self.addToCombines(test)
629: sys.stdout.write("Combines: %d\n" % len(self.combinesRan))
630: sys.stdout.write("%s\n" % self.combinesRan.keys())
631: return
632:
633: if options.enableLog:
634: self.logFile = open(options.logFile, "w")
635: try:
636: for test in self.testList:
637: self.counters[self.CNT_TOTAL] += 1
638: #
639: # Filter tests.
640: #
641: if options.singleTest is not None and options.singleTest != "":
642: if (test.name != options.singleTest):
643: continue
644: elif options.combines is not None:
645: if not options.combines.__contains__(test.combineName):
646: continue
647: elif options.testStartsWith is not None:
648: if not test.name.startswith(options.testStartsWith):
649: continue
650: elif options.combineStartsWith is not None:
651: if not test.combineName.startswith(options.combineStartsWith):
652: continue
653:
654: if options.maxTestCount != -1 and self.counters[self.CNT_RAN] >= options.maxTestCount:
655: break
656: self.counters[self.CNT_RAN] += 1
657: #
658: # Run the thing, dammit.
659: #
660: try:
661: test.setUp()
662: try:
663: test.run()
664: finally:
665: test.tearDown()
666: finally:
667: #
668: # Evaluate.
669: #
670: test.finalize()
671: self.reportTest(test)
672: if options.reportCombines or options.reportErrCombines:
673: self.addToCombines(test)
674: self.counters = self.updateCounters(test, self.counters)
675: finally:
676: if options.reportCombines or options.reportErrCombines:
677: #
678: # Build a report for every single combine.
679: #
680: # TODO: How to sort a dict?
681: #
682: self.combinesRan.keys().sort(None)
683: for key in self.combinesRan.keys():
684: self.reportCombine(key)
685:
686: #
687: # Display the final report.
688: #
689: if options.silent:
690: self.displayShortResults(sys.stdout, True, None, self.counters)
691: else:
692: sys.stdout.write("===========================\n")
693: self.displayResults(sys.stdout, True, None, self.counters)
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to xstc.py CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / libxml2 / xstc