![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to test_runner.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / strongswan / src / libstrongswan / tests|
Return to test_runner.c CVS log | Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / strongswan / src / libstrongswan / tests |
1.1 misho 1: /*
2: * Copyright (C) 2013 Tobias Brunner
3: * HSR Hochschule fuer Technik Rapperswil
4: * Copyright (C) 2013 Martin Willi
5: * Copyright (C) 2013 revosec AG
6: *
7: * This program is free software; you can redistribute it and/or modify it
8: * under the terms of the GNU General Public License as published by the
9: * Free Software Foundation; either version 2 of the License, or (at your
10: * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11: *
12: * This program is distributed in the hope that it will be useful, but
13: * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14: * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15: * for more details.
16: */
17:
18: #include "test_runner.h"
19:
20: #include <library.h>
21: #include <threading/thread.h>
22: #include <plugins/plugin_feature.h>
23: #include <collections/array.h>
24: #include <utils/test.h>
25:
26: #include <stdlib.h>
27: #include <dirent.h>
28: #include <unistd.h>
29: #include <limits.h>
30: #include <stdlib.h>
31:
32: /**
33: * Get a tty color escape character for stderr
34: */
35: #define TTY(color) tty_escape_get(2, TTY_FG_##color)
36:
37: /**
38: * A global symbol indicating libtest linkage
39: */
40: #ifdef WIN32
41: __declspec(dllexport)
42: #endif
43: bool test_runner_available = TRUE;
44:
45: /**
46: * Destroy a single test suite and associated data
47: */
48: static void destroy_suite(test_suite_t *suite)
49: {
50: test_case_t *tcase;
51:
52: while (array_remove(suite->tcases, 0, &tcase))
53: {
54: array_destroy(tcase->functions);
55: array_destroy(tcase->fixtures);
56: }
57: free(suite);
58: }
59:
60: /**
61: * Filter loaded test suites, either remove suites listed (exclude=TRUE), or all
62: * that are not listed (exclude=FALSE).
63: */
64: static void apply_filter(array_t *loaded, char *filter, bool exclude)
65: {
66: enumerator_t *enumerator, *names;
67: hashtable_t *listed;
68: test_suite_t *suite;
69: char *name;
70:
71: listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
72: names = enumerator_create_token(filter, ",", " ");
73: while (names->enumerate(names, &name))
74: {
75: listed->put(listed, name, name);
76: }
77: enumerator = array_create_enumerator(loaded);
78: while (enumerator->enumerate(enumerator, &suite))
79: {
80: if ((exclude && listed->get(listed, suite->name)) ||
81: (!exclude && !listed->get(listed, suite->name)))
82: {
83: array_remove_at(loaded, enumerator);
84: destroy_suite(suite);
85: }
86: }
87: enumerator->destroy(enumerator);
88: listed->destroy(listed);
89: names->destroy(names);
90: }
91:
92: /**
93: * Check if the given string is contained in the filter string.
94: */
95: static bool is_in_filter(const char *find, char *filter)
96: {
97: enumerator_t *names;
98: bool found = FALSE;
99: char *name;
100:
101: names = enumerator_create_token(filter, ",", " ");
102: while (names->enumerate(names, &name))
103: {
104: if (streq(name, find))
105: {
106: found = TRUE;
107: break;
108: }
109: }
110: names->destroy(names);
111: return found;
112: }
113:
114: /**
115: * Removes and destroys test suites that are not selected or
116: * explicitly excluded.
117: */
118: static void filter_suites(array_t *loaded)
119: {
120: char *filter;
121:
122: filter = getenv("TESTS_SUITES");
123: if (filter)
124: {
125: apply_filter(loaded, filter, FALSE);
126: }
127: filter = getenv("TESTS_SUITES_EXCLUDE");
128: if (filter)
129: {
130: apply_filter(loaded, filter, TRUE);
131: }
132: }
133:
134: /**
135: * Load all available test suites, or optionally only selected ones.
136: */
137: static array_t *load_suites(test_configuration_t configs[],
138: test_runner_init_t init, char *cfg)
139: {
140: array_t *suites;
141: bool old = FALSE;
142: int i;
143:
144: library_init(cfg, "test-runner");
145:
146: test_setup_handler();
147:
148: if (init && !init(TRUE))
149: {
150: library_deinit();
151: return NULL;
152: }
153: lib->plugins->status(lib->plugins, LEVEL_CTRL);
154:
155: if (lib->leak_detective)
156: {
157: old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
158: }
159:
160: suites = array_create(0, 0);
161:
162: for (i = 0; configs[i].suite; i++)
163: {
164: if (configs[i].feature.type == 0 ||
165: lib->plugins->has_feature(lib->plugins, configs[i].feature))
166: {
167: array_insert(suites, -1, configs[i].suite());
168: }
169: }
170: filter_suites(suites);
171:
172: if (lib->leak_detective)
173: {
174: lib->leak_detective->set_state(lib->leak_detective, old);
175: }
176:
177: if (init)
178: {
179: init(FALSE);
180: }
181: library_deinit();
182:
183: return suites;
184: }
185:
186: /**
187: * Unload and destroy test suites and associated data
188: */
189: static void unload_suites(array_t *suites)
190: {
191: test_suite_t *suite;
192:
193: while (array_remove(suites, 0, &suite))
194: {
195: destroy_suite(suite);
196: }
197: array_destroy(suites);
198: }
199:
200: /**
201: * Run a single test function, return FALSE on failure
202: */
203: static bool run_test(test_function_t *tfun, int i)
204: {
205: if (test_restore_point())
206: {
207: tfun->cb(i);
208: return TRUE;
209: }
210: thread_cleanup_popall();
211: return FALSE;
212: }
213:
214: /**
215: * Invoke fixture setup/teardown
216: */
217: static bool call_fixture(test_case_t *tcase, bool up)
218: {
219: enumerator_t *enumerator;
220: test_fixture_t *fixture;
221: bool failure = FALSE;
222:
223: enumerator = array_create_enumerator(tcase->fixtures);
224: while (enumerator->enumerate(enumerator, &fixture))
225: {
226: if (test_restore_point())
227: {
228: if (up)
229: {
230: if (fixture->setup)
231: {
232: fixture->setup();
233: }
234: }
235: else
236: {
237: if (fixture->teardown)
238: {
239: fixture->teardown();
240: }
241: }
242: }
243: else
244: {
245: thread_cleanup_popall();
246: failure = TRUE;
247: break;
248: }
249: }
250: enumerator->destroy(enumerator);
251:
252: return !failure;
253: }
254:
255: /**
256: * Test initialization, initializes libstrongswan for the next run
257: */
258: static bool pre_test(test_runner_init_t init, char *cfg)
259: {
260: library_init(cfg, "test-runner");
261:
262: /* use non-blocking RNG to generate keys fast */
263: lib->settings->set_default_str(lib->settings,
264: "libstrongswan.plugins.random.random",
265: lib->settings->get_str(lib->settings,
266: "libstrongswan.plugins.random.urandom", "/dev/urandom"));
267: /* same for the gcrypt plugin */
268: lib->settings->set_default_str(lib->settings,
269: "libstrongswan.plugins.gcrypt.quick_random", "yes");
270:
271: if (lib->leak_detective)
272: {
273: /* disable leak reports during testing */
274: lib->leak_detective->set_report_cb(lib->leak_detective,
275: NULL, NULL, NULL);
276: }
277: if (init && !init(TRUE))
278: {
279: library_deinit();
280: return FALSE;
281: }
282: return TRUE;
283: }
284:
285: /**
286: * Failure description
287: */
288: typedef struct {
289: char *name;
290: char msg[4096 - sizeof(char*) - 2 * sizeof(int)];
291: const char *file;
292: int line;
293: int i;
294: backtrace_t *bt;
295: } failure_t;
296:
297: /**
298: * Data passed to leak report callbacks
299: */
300: typedef struct {
301: array_t *failures;
302: char *name;
303: int i;
304: int leaks;
305: } report_data_t;
306:
307: /**
308: * Leak report callback, build failures from leaks
309: */
310: static void report_leaks(report_data_t *data, int count, size_t bytes,
311: backtrace_t *bt, bool detailed)
312: {
313: failure_t failure = {
314: .name = data->name,
315: .i = data->i,
316: .bt = bt->clone(bt),
317: };
318:
319: snprintf(failure.msg, sizeof(failure.msg),
320: "Leak detected: %d allocations using %zu bytes", count, bytes);
321:
322: array_insert(data->failures, -1, &failure);
323: }
324:
325: /**
326: * Leak summary callback, check if any leaks found
327: */
328: static void sum_leaks(report_data_t *data, int count, size_t bytes,
329: int whitelisted)
330: {
331: data->leaks = count;
332: }
333:
334: /**
335: * Do library cleanup and optionally check for memory leaks
336: */
337: static bool post_test(test_runner_init_t init, bool check_leaks,
338: array_t *failures, char *name, int i, int *leaks)
339: {
340: report_data_t data = {
341: .failures = failures,
342: .name = name,
343: .i = i,
344: };
345:
346: if (init)
347: {
348: if (test_restore_point())
349: {
350: init(FALSE);
351: }
352: else
353: {
354: thread_cleanup_popall();
355: library_deinit();
356: return FALSE;
357: }
358: }
359: if (check_leaks && lib->leak_detective)
360: {
361: lib->leak_detective->set_report_cb(lib->leak_detective,
362: (leak_detective_report_cb_t)report_leaks,
363: (leak_detective_summary_cb_t)sum_leaks, &data);
364: }
365: library_deinit();
366:
367: *leaks = data.leaks;
368: return TRUE;
369: }
370:
371: /**
372: * Collect failure information, add failure_t to array
373: */
374: static void collect_failure_info(array_t *failures, char *name, int i)
375: {
376: failure_t failure = {
377: .name = name,
378: .i = i,
379: .bt = test_failure_backtrace(),
380: };
381:
382: failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
383: &failure.file);
384:
385: array_insert(failures, -1, &failure);
386: }
387:
388: /**
389: * Collect warning information, add failure_t to array
390: */
391: static bool collect_warning_info(array_t *warnings, char *name, int i)
392: {
393: failure_t warning = {
394: .name = name,
395: .i = i,
396: };
397:
398: warning.line = test_warning_get(warning.msg, sizeof(warning.msg),
399: &warning.file);
400: if (warning.line)
401: {
402: array_insert(warnings, -1, &warning);
403: }
404: return warning.line;
405: }
406:
407: /**
408: * Print array of collected failure_t to stderr
409: */
410: static void print_failures(array_t *failures, bool warnings)
411: {
412: failure_t failure;
413:
414: threads_init();
415: backtrace_init();
416:
417: while (array_remove(failures, 0, &failure))
418: {
419: if (warnings)
420: {
421: fprintf(stderr, " %sWarning in '%s': %s (",
422: TTY(YELLOW), failure.name, failure.msg);
423: }
424: else
425: {
426: fprintf(stderr, " %sFailure in '%s': %s (",
427: TTY(RED), failure.name, failure.msg);
428: }
429: if (failure.line)
430: {
431: fprintf(stderr, "%s:%d, ", failure.file, failure.line);
432: }
433: fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
434: if (failure.bt)
435: {
436: failure.bt->log(failure.bt, stderr, TRUE);
437: failure.bt->destroy(failure.bt);
438: }
439: }
440:
441: backtrace_deinit();
442: threads_deinit();
443: }
444:
445: /**
446: * Run a single test case with fixtures
447: */
448: static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
449: {
450: enumerator_t *enumerator;
451: test_function_t *tfun;
452: int passed = 0;
453: array_t *failures, *warnings;
454:
455: failures = array_create(sizeof(failure_t), 0);
456: warnings = array_create(sizeof(failure_t), 0);
457:
458: fprintf(stderr, " Running case '%s': ", tcase->name);
459: fflush(stderr);
460:
461: enumerator = array_create_enumerator(tcase->functions);
462: while (enumerator->enumerate(enumerator, &tfun))
463: {
464: int i, rounds = 0;
465:
466: for (i = tfun->start; i < tfun->end; i++)
467: {
468: if (pre_test(init, cfg))
469: {
470: bool ok = FALSE;
471: int leaks = 0;
472:
473: test_setup_timeout(tcase->timeout);
474:
475: if (call_fixture(tcase, TRUE))
476: {
477: if (run_test(tfun, i))
478: {
479: if (call_fixture(tcase, FALSE))
480: {
481: ok = TRUE;
482: }
483: }
484: else
485: {
486: call_fixture(tcase, FALSE);
487: }
488: }
489: if (!post_test(init, ok, failures, tfun->name, i, &leaks))
490: {
491: ok = FALSE;
492: }
493:
494: test_setup_timeout(0);
495:
496: if (ok)
497: {
498: if (!leaks)
499: {
500: rounds++;
501: if (!collect_warning_info(warnings, tfun->name, i))
502: {
503: fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
504: }
505: else
506: {
507: fprintf(stderr, "%s~%s", TTY(YELLOW), TTY(DEF));
508: }
509: }
510: }
511: else
512: {
513: collect_failure_info(failures, tfun->name, i);
514: }
515: if (!ok || leaks)
516: {
517: fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
518: }
519: }
520: else
521: {
522: fprintf(stderr, "!");
523: }
524: }
525: fflush(stderr);
526: if (rounds == tfun->end - tfun->start)
527: {
528: passed++;
529: }
530: }
531: enumerator->destroy(enumerator);
532:
533: fprintf(stderr, "\n");
534:
535: print_failures(warnings, TRUE);
536: print_failures(failures, FALSE);
537: array_destroy(failures);
538: array_destroy(warnings);
539:
540: return passed == array_count(tcase->functions);
541: }
542:
543: /**
544: * Run a single test suite
545: */
546: static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
547: {
548: enumerator_t *enumerator;
549: test_case_t *tcase;
550: int passed = 0;
551:
552: fprintf(stderr, " Running suite '%s':\n", suite->name);
553:
554: enumerator = array_create_enumerator(suite->tcases);
555: while (enumerator->enumerate(enumerator, &tcase))
556: {
557: if (run_case(tcase, init, cfg))
558: {
559: passed++;
560: }
561: }
562: enumerator->destroy(enumerator);
563:
564: if (passed == array_count(suite->tcases))
565: {
566: fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
567: TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
568: return TRUE;
569: }
570: fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
571: TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
572: return FALSE;
573: }
574:
575: /**
576: * See header.
577: */
578: int test_runner_run(const char *name, test_configuration_t configs[],
579: test_runner_init_t init)
580: {
581: array_t *suites;
582: test_suite_t *suite;
583: enumerator_t *enumerator;
584: int passed = 0, result;
585: level_t level = LEVEL_SILENT;
586: char *cfg, *runners, *verbosity;
587:
588: /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
589: dup2(2, 1);
590:
591: runners = getenv("TESTS_RUNNERS");
592: if (runners && !is_in_filter(name, runners))
593: {
594: return EXIT_SUCCESS;
595: }
596:
597: cfg = getenv("TESTS_STRONGSWAN_CONF");
598:
599: suites = load_suites(configs, init, cfg);
600: if (!suites)
601: {
602: return EXIT_FAILURE;
603: }
604:
605: verbosity = getenv("TESTS_VERBOSITY");
606: if (verbosity)
607: {
608: level = atoi(verbosity);
609: }
610: dbg_default_set_level(level);
611:
612: fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
613:
614: enumerator = array_create_enumerator(suites);
615: while (enumerator->enumerate(enumerator, &suite))
616: {
617: if (run_suite(suite, init, cfg))
618: {
619: passed++;
620: }
621: }
622: enumerator->destroy(enumerator);
623:
624: if (passed == array_count(suites))
625: {
626: fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
627: TTY(GREEN), array_count(suites), name, TTY(DEF));
628: result = EXIT_SUCCESS;
629: }
630: else
631: {
632: fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
633: TTY(RED), passed, array_count(suites), name, TTY(DEF));
634: result = EXIT_FAILURE;
635: }
636:
637: unload_suites(suites);
638:
639: return result;
640: }