1: #!/usr/bin/python -u
2: import glob, os, string, sys, thread, time
3: # import difflib
4: import libxml2
5:
6: ###
7: #
8: # This is a "Work in Progress" attempt at a python script to run the
9: # various regression tests. The rationale for this is that it should be
10: # possible to run this on most major platforms, including those (such as
11: # Windows) which don't support gnu Make.
12: #
13: # The script is driven by a parameter file which defines the various tests
14: # to be run, together with the unique settings for each of these tests. A
15: # script for Linux is included (regressions.xml), with comments indicating
16: # the significance of the various parameters. To run the tests under Windows,
17: # edit regressions.xml and remove the comment around the default parameter
18: # "<execpath>" (i.e. make it point to the location of the binary executables).
19: #
20: # Note that this current version requires the Python bindings for libxml2 to
21: # have been previously installed and accessible
22: #
23: # See Copyright for the status of this software.
24: # William Brack (wbrack@mmm.com.hk)
25: #
26: ###
27: defaultParams = {} # will be used as a dictionary to hold the parsed params
28:
29: # This routine is used for comparing the expected stdout / stdin with the results.
30: # The expected data has already been read in; the result is a file descriptor.
31: # Within the two sets of data, lines may begin with a path string. If so, the
32: # code "relativises" it by removing the path component. The first argument is a
33: # list already read in by a separate thread; the second is a file descriptor.
34: # The two 'base' arguments are to let me "relativise" the results files, allowing
35: # the script to be run from any directory.
36: def compFiles(res, expected, base1, base2):
37: l1 = len(base1)
38: exp = expected.readlines()
39: expected.close()
40: # the "relativisation" is done here
41: for i in range(len(res)):
42: j = string.find(res[i],base1)
43: if (j == 0) or ((j == 2) and (res[i][0:2] == './')):
44: col = string.find(res[i],':')
45: if col > 0:
46: start = string.rfind(res[i][:col], '/')
47: if start > 0:
48: res[i] = res[i][start+1:]
49:
50: for i in range(len(exp)):
51: j = string.find(exp[i],base2)
52: if (j == 0) or ((j == 2) and (exp[i][0:2] == './')):
53: col = string.find(exp[i],':')
54: if col > 0:
55: start = string.rfind(exp[i][:col], '/')
56: if start > 0:
57: exp[i] = exp[i][start+1:]
58:
59: ret = 0
60: # ideally we would like to use difflib functions here to do a
61: # nice comparison of the two sets. Unfortunately, during testing
62: # (using python 2.3.3 and 2.3.4) the following code went into
63: # a dead loop under windows. I'll pursue this later.
64: # diff = difflib.ndiff(res, exp)
65: # diff = list(diff)
66: # for line in diff:
67: # if line[:2] != ' ':
68: # print string.strip(line)
69: # ret = -1
70:
71: # the following simple compare is fine for when the two data sets
72: # (actual result vs. expected result) are equal, which should be true for
73: # us. Unfortunately, if the test fails it's not nice at all.
74: rl = len(res)
75: el = len(exp)
76: if el != rl:
77: print 'Length of expected is %d, result is %d' % (el, rl)
78: ret = -1
79: for i in range(min(el, rl)):
80: if string.strip(res[i]) != string.strip(exp[i]):
81: print '+:%s-:%s' % (res[i], exp[i])
82: ret = -1
83: if el > rl:
84: for i in range(rl, el):
85: print '-:%s' % exp[i]
86: ret = -1
87: elif rl > el:
88: for i in range (el, rl):
89: print '+:%s' % res[i]
90: ret = -1
91: return ret
92:
93: # Separate threads to handle stdout and stderr are created to run this function
94: def readPfile(file, list, flag):
95: data = file.readlines() # no call by reference, so I cheat
96: for l in data:
97: list.append(l)
98: file.close()
99: flag.append('ok')
100:
101: # This routine runs the test program (e.g. xmllint)
102: def runOneTest(testDescription, filename, inbase, errbase):
103: if 'execpath' in testDescription:
104: dir = testDescription['execpath'] + '/'
105: else:
106: dir = ''
107: cmd = os.path.abspath(dir + testDescription['testprog'])
108: if 'flag' in testDescription:
109: for f in string.split(testDescription['flag']):
110: cmd += ' ' + f
111: if 'stdin' not in testDescription:
112: cmd += ' ' + inbase + filename
113: if 'extarg' in testDescription:
114: cmd += ' ' + testDescription['extarg']
115:
116: noResult = 0
117: expout = None
118: if 'resext' in testDescription:
119: if testDescription['resext'] == 'None':
120: noResult = 1
121: else:
122: ext = '.' + testDescription['resext']
123: else:
124: ext = ''
125: if not noResult:
126: try:
127: fname = errbase + filename + ext
128: expout = open(fname, 'rt')
129: except:
130: print "Can't open result file %s - bypassing test" % fname
131: return
132:
133: noErrors = 0
134: if 'reserrext' in testDescription:
135: if testDescription['reserrext'] == 'None':
136: noErrors = 1
137: else:
138: if len(testDescription['reserrext'])>0:
139: ext = '.' + testDescription['reserrext']
140: else:
141: ext = ''
142: else:
143: ext = ''
144: if not noErrors:
145: try:
146: fname = errbase + filename + ext
147: experr = open(fname, 'rt')
148: except:
149: experr = None
150: else:
151: experr = None
152:
153: pin, pout, perr = os.popen3(cmd)
154: if 'stdin' in testDescription:
155: infile = open(inbase + filename, 'rt')
156: pin.writelines(infile.readlines())
157: infile.close()
158: pin.close()
159:
160: # popen is great fun, but can lead to the old "deadly embrace", because
161: # synchronizing the writing (by the task being run) of stdout and stderr
162: # with respect to the reading (by this task) is basically impossible. I
163: # tried several ways to cheat, but the only way I have found which works
164: # is to do a *very* elementary multi-threading approach. We can only hope
165: # that Python threads are implemented on the target system (it's okay for
166: # Linux and Windows)
167:
168: th1Flag = [] # flags to show when threads finish
169: th2Flag = []
170: outfile = [] # lists to contain the pipe data
171: errfile = []
172: th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag))
173: th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag))
174: while (len(th1Flag)==0) or (len(th2Flag)==0):
175: time.sleep(0.001)
176: if not noResult:
177: ret = compFiles(outfile, expout, inbase, 'test/')
178: if ret != 0:
179: print 'trouble with %s' % cmd
180: else:
181: if len(outfile) != 0:
182: for l in outfile:
183: print l
184: print 'trouble with %s' % cmd
185: if experr != None:
186: ret = compFiles(errfile, experr, inbase, 'test/')
187: if ret != 0:
188: print 'trouble with %s' % cmd
189: else:
190: if not noErrors:
191: if len(errfile) != 0:
192: for l in errfile:
193: print l
194: print 'trouble with %s' % cmd
195:
196: if 'stdin' not in testDescription:
197: pin.close()
198:
199: # This routine is called by the parameter decoding routine whenever the end of a
200: # 'test' section is encountered. Depending upon file globbing, a large number of
201: # individual tests may be run.
202: def runTest(description):
203: testDescription = defaultParams.copy() # set defaults
204: testDescription.update(description) # override with current ent
205: if 'testname' in testDescription:
206: print "## %s" % testDescription['testname']
207: if not 'file' in testDescription:
208: print "No file specified - can't run this test!"
209: return
210: # Set up the source and results directory paths from the decoded params
211: dir = ''
212: if 'srcdir' in testDescription:
213: dir += testDescription['srcdir'] + '/'
214: if 'srcsub' in testDescription:
215: dir += testDescription['srcsub'] + '/'
216:
217: rdir = ''
218: if 'resdir' in testDescription:
219: rdir += testDescription['resdir'] + '/'
220: if 'ressub' in testDescription:
221: rdir += testDescription['ressub'] + '/'
222:
223: testFiles = glob.glob(os.path.abspath(dir + testDescription['file']))
224: if testFiles == []:
225: print "No files result from '%s'" % testDescription['file']
226: return
227:
228: # Some test programs just don't work (yet). For now we exclude them.
229: count = 0
230: excl = []
231: if 'exclfile' in testDescription:
232: for f in string.split(testDescription['exclfile']):
233: glb = glob.glob(dir + f)
234: for g in glb:
235: excl.append(os.path.abspath(g))
236:
237: # Run the specified test program
238: for f in testFiles:
239: if not os.path.isdir(f):
240: if f not in excl:
241: count = count + 1
242: runOneTest(testDescription, os.path.basename(f), dir, rdir)
243:
244: #
245: # The following classes are used with the xmlreader interface to interpret the
246: # parameter file. Once a test section has been identified, runTest is called
247: # with a dictionary containing the parsed results of the interpretation.
248: #
249:
250: class testDefaults:
251: curText = '' # accumulates text content of parameter
252:
253: def addToDict(self, key):
254: txt = string.strip(self.curText)
255: # if txt == '':
256: # return
257: if key not in defaultParams:
258: defaultParams[key] = txt
259: else:
260: defaultParams[key] += ' ' + txt
261:
262: def processNode(self, reader, curClass):
263: if reader.Depth() == 2:
264: if reader.NodeType() == 1:
265: self.curText = '' # clear the working variable
266: elif reader.NodeType() == 15:
267: if (reader.Name() != '#text') and (reader.Name() != '#comment'):
268: self.addToDict(reader.Name())
269: elif reader.Depth() == 3:
270: if reader.Name() == '#text':
271: self.curText += reader.Value()
272:
273: elif reader.NodeType() == 15: # end of element
274: print "Defaults have been set to:"
275: for k in defaultParams.keys():
276: print " %s : '%s'" % (k, defaultParams[k])
277: curClass = rootClass()
278: return curClass
279:
280:
281: class testClass:
282: def __init__(self):
283: self.testParams = {} # start with an empty set of params
284: self.curText = '' # and empty text
285:
286: def addToDict(self, key):
287: data = string.strip(self.curText)
288: if key not in self.testParams:
289: self.testParams[key] = data
290: else:
291: if self.testParams[key] != '':
292: data = ' ' + data
293: self.testParams[key] += data
294:
295: def processNode(self, reader, curClass):
296: if reader.Depth() == 2:
297: if reader.NodeType() == 1:
298: self.curText = '' # clear the working variable
299: if reader.Name() not in self.testParams:
300: self.testParams[reader.Name()] = ''
301: elif reader.NodeType() == 15:
302: if (reader.Name() != '#text') and (reader.Name() != '#comment'):
303: self.addToDict(reader.Name())
304: elif reader.Depth() == 3:
305: if reader.Name() == '#text':
306: self.curText += reader.Value()
307:
308: elif reader.NodeType() == 15: # end of element
309: runTest(self.testParams)
310: curClass = rootClass()
311: return curClass
312:
313:
314: class rootClass:
315: def processNode(self, reader, curClass):
316: if reader.Depth() == 0:
317: return curClass
318: if reader.Depth() != 1:
319: print "Unexpected junk: Level %d, type %d, name %s" % (
320: reader.Depth(), reader.NodeType(), reader.Name())
321: return curClass
322: if reader.Name() == 'test':
323: curClass = testClass()
324: curClass.testParams = {}
325: elif reader.Name() == 'defaults':
326: curClass = testDefaults()
327: return curClass
328:
329: def streamFile(filename):
330: try:
331: reader = libxml2.newTextReaderFilename(filename)
332: except:
333: print "unable to open %s" % (filename)
334: return
335:
336: curClass = rootClass()
337: ret = reader.Read()
338: while ret == 1:
339: curClass = curClass.processNode(reader, curClass)
340: ret = reader.Read()
341:
342: if ret != 0:
343: print "%s : failed to parse" % (filename)
344:
345: # OK, we're finished with all the routines. Now for the main program:-
346: if len(sys.argv) != 2:
347: print "Usage: maketest {filename}"
348: sys.exit(-1)
349:
350: streamFile(sys.argv[1])
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to regressions.py CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / libxml2