![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to test_runner.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / strongswan / src / libstrongswan / tests|
Return to test_runner.c CVS log | Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / strongswan / src / libstrongswan / tests |
1.1 misho 1: /*
2: * Copyright (C) 2013 Tobias Brunner
3: * HSR Hochschule fuer Technik Rapperswil
4: * Copyright (C) 2013 Martin Willi
5: * Copyright (C) 2013 revosec AG
6: *
7: * This program is free software; you can redistribute it and/or modify it
8: * under the terms of the GNU General Public License as published by the
9: * Free Software Foundation; either version 2 of the License, or (at your
10: * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11: *
12: * This program is distributed in the hope that it will be useful, but
13: * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14: * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15: * for more details.
16: */
17:
18: #include "test_runner.h"
19:
20: #include <library.h>
21: #include <threading/thread.h>
22: #include <plugins/plugin_feature.h>
23: #include <collections/array.h>
24: #include <utils/test.h>
25:
26: #include <stdlib.h>
27: #include <dirent.h>
28: #include <unistd.h>
29: #include <limits.h>
30: #include <stdlib.h>
1.1.1.2 ! misho 31: #include <time.h>
1.1 misho 32:
33: /**
34: * Get a tty color escape character for stderr
35: */
36: #define TTY(color) tty_escape_get(2, TTY_FG_##color)
37:
38: /**
39: * A global symbol indicating libtest linkage
40: */
41: #ifdef WIN32
42: __declspec(dllexport)
43: #endif
44: bool test_runner_available = TRUE;
45:
46: /**
1.1.1.2 ! misho 47: * Destroy data associated with a test case.
1.1 misho 48: */
1.1.1.2 ! misho 49: static void destroy_case(test_case_t *tcase)
1.1 misho 50: {
1.1.1.2 ! misho 51: array_destroy(tcase->functions);
! 52: array_destroy(tcase->fixtures);
! 53: }
1.1 misho 54:
1.1.1.2 ! misho 55: /**
! 56: * Destroy a single test suite and associated data.
! 57: */
! 58: static void destroy_suite(test_suite_t *suite)
! 59: {
! 60: array_destroy_function(suite->tcases, (void*)destroy_case, NULL);
1.1 misho 61: free(suite);
62: }
63:
64: /**
1.1.1.2 ! misho 65: * Identifies on which component to apply the given filter.
1.1 misho 66: */
1.1.1.2 ! misho 67: typedef enum {
! 68: FILTER_SUITES,
! 69: FILTER_CASES,
! 70: FILTER_FUNCTIONS,
! 71: } filter_component_t;
! 72:
! 73: /**
! 74: * Check if the component with the given name should be filtered/removed.
! 75: */
! 76: static bool filter_name(const char *name, hashtable_t *names, bool exclude)
1.1 misho 77: {
1.1.1.2 ! misho 78: return (exclude && names->get(names, name)) ||
! 79: (!exclude && !names->get(names, name));
! 80: }
! 81:
! 82: /**
! 83: * Filter loaded test suites/cases/functions, either remove components listed
! 84: * (exclude=TRUE), or all that are not listed (exclude=FALSE).
! 85: * Empty test cases/suites are removed and destroyed.
! 86: */
! 87: static void apply_filter(array_t *loaded, filter_component_t comp, char *filter,
! 88: bool exclude)
! 89: {
! 90: enumerator_t *enumerator, *tcases, *functions, *names;
1.1 misho 91: hashtable_t *listed;
92: test_suite_t *suite;
1.1.1.2 ! misho 93: test_case_t *tcase;
! 94: test_function_t *func;
1.1 misho 95: char *name;
96:
97: listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
98: names = enumerator_create_token(filter, ",", " ");
99: while (names->enumerate(names, &name))
100: {
101: listed->put(listed, name, name);
102: }
1.1.1.2 ! misho 103:
1.1 misho 104: enumerator = array_create_enumerator(loaded);
105: while (enumerator->enumerate(enumerator, &suite))
106: {
1.1.1.2 ! misho 107: if (comp == FILTER_SUITES)
! 108: {
! 109: if (filter_name(suite->name, listed, exclude))
! 110: {
! 111: array_remove_at(loaded, enumerator);
! 112: destroy_suite(suite);
! 113: }
! 114: continue;
! 115: }
! 116: tcases = array_create_enumerator(suite->tcases);
! 117: while (tcases->enumerate(tcases, &tcase))
! 118: {
! 119: if (comp == FILTER_CASES)
! 120: {
! 121: if (filter_name(tcase->name, listed, exclude))
! 122: {
! 123: array_remove_at(suite->tcases, tcases);
! 124: destroy_case(tcase);
! 125: }
! 126: continue;
! 127: }
! 128: functions = array_create_enumerator(tcase->functions);
! 129: while (functions->enumerate(functions, &func))
! 130: {
! 131: if (filter_name(func->name, listed, exclude))
! 132: {
! 133: array_remove_at(tcase->functions, functions);
! 134: }
! 135: }
! 136: functions->destroy(functions);
! 137:
! 138: if (!array_count(tcase->functions))
! 139: {
! 140: array_remove_at(suite->tcases, tcases);
! 141: destroy_case(tcase);
! 142: }
! 143: }
! 144: tcases->destroy(tcases);
! 145:
! 146: if (!array_count(suite->tcases))
1.1 misho 147: {
148: array_remove_at(loaded, enumerator);
149: destroy_suite(suite);
150: }
151: }
152: enumerator->destroy(enumerator);
153: listed->destroy(listed);
154: names->destroy(names);
155: }
156:
157: /**
158: * Check if the given string is contained in the filter string.
159: */
160: static bool is_in_filter(const char *find, char *filter)
161: {
162: enumerator_t *names;
163: bool found = FALSE;
164: char *name;
165:
166: names = enumerator_create_token(filter, ",", " ");
167: while (names->enumerate(names, &name))
168: {
169: if (streq(name, find))
170: {
171: found = TRUE;
172: break;
173: }
174: }
175: names->destroy(names);
176: return found;
177: }
178:
179: /**
1.1.1.2 ! misho 180: * Removes and destroys test suites/cases/functions that are not selected or
! 181: * explicitly excluded. Takes names of two environment variables.
1.1 misho 182: */
1.1.1.2 ! misho 183: static void filter_components(array_t *loaded, filter_component_t comp,
! 184: char *sel, char *exc)
1.1 misho 185: {
186: char *filter;
187:
1.1.1.2 ! misho 188: filter = getenv(sel);
1.1 misho 189: if (filter)
190: {
1.1.1.2 ! misho 191: apply_filter(loaded, comp, filter, FALSE);
1.1 misho 192: }
1.1.1.2 ! misho 193: filter = getenv(exc);
1.1 misho 194: if (filter)
195: {
1.1.1.2 ! misho 196: apply_filter(loaded, comp, filter, TRUE);
1.1 misho 197: }
198: }
199:
200: /**
201: * Load all available test suites, or optionally only selected ones.
202: */
203: static array_t *load_suites(test_configuration_t configs[],
204: test_runner_init_t init, char *cfg)
205: {
206: array_t *suites;
207: bool old = FALSE;
208: int i;
209:
210: library_init(cfg, "test-runner");
211:
212: test_setup_handler();
213:
214: if (init && !init(TRUE))
215: {
216: library_deinit();
217: return NULL;
218: }
219: lib->plugins->status(lib->plugins, LEVEL_CTRL);
220:
221: if (lib->leak_detective)
222: {
223: old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
224: }
225:
226: suites = array_create(0, 0);
227:
228: for (i = 0; configs[i].suite; i++)
229: {
230: if (configs[i].feature.type == 0 ||
231: lib->plugins->has_feature(lib->plugins, configs[i].feature))
232: {
233: array_insert(suites, -1, configs[i].suite());
234: }
235: }
1.1.1.2 ! misho 236: filter_components(suites, FILTER_SUITES, "TESTS_SUITES",
! 237: "TESTS_SUITES_EXCLUDE");
! 238: filter_components(suites, FILTER_CASES, "TESTS_CASES",
! 239: "TESTS_CASES_EXCLUDE");
! 240: filter_components(suites, FILTER_FUNCTIONS, "TESTS_FUNCTIONS",
! 241: "TESTS_FUNCTIONS_EXCLUDE");
1.1 misho 242:
243: if (lib->leak_detective)
244: {
245: lib->leak_detective->set_state(lib->leak_detective, old);
246: }
247:
248: if (init)
249: {
250: init(FALSE);
251: }
252: library_deinit();
253:
254: return suites;
255: }
256:
257: /**
258: * Unload and destroy test suites and associated data
259: */
260: static void unload_suites(array_t *suites)
261: {
262: test_suite_t *suite;
263:
264: while (array_remove(suites, 0, &suite))
265: {
266: destroy_suite(suite);
267: }
268: array_destroy(suites);
269: }
270:
271: /**
272: * Run a single test function, return FALSE on failure
273: */
274: static bool run_test(test_function_t *tfun, int i)
275: {
276: if (test_restore_point())
277: {
278: tfun->cb(i);
279: return TRUE;
280: }
281: thread_cleanup_popall();
282: return FALSE;
283: }
284:
285: /**
286: * Invoke fixture setup/teardown
287: */
1.1.1.2 ! misho 288: static bool call_fixture(test_case_t *tcase, bool up, int i)
1.1 misho 289: {
290: enumerator_t *enumerator;
291: test_fixture_t *fixture;
292: bool failure = FALSE;
293:
294: enumerator = array_create_enumerator(tcase->fixtures);
295: while (enumerator->enumerate(enumerator, &fixture))
296: {
297: if (test_restore_point())
298: {
299: if (up)
300: {
301: if (fixture->setup)
302: {
1.1.1.2 ! misho 303: fixture->setup(i);
1.1 misho 304: }
305: }
306: else
307: {
308: if (fixture->teardown)
309: {
1.1.1.2 ! misho 310: fixture->teardown(i);
1.1 misho 311: }
312: }
313: }
314: else
315: {
316: thread_cleanup_popall();
317: failure = TRUE;
318: break;
319: }
320: }
321: enumerator->destroy(enumerator);
322:
323: return !failure;
324: }
325:
326: /**
327: * Test initialization, initializes libstrongswan for the next run
328: */
329: static bool pre_test(test_runner_init_t init, char *cfg)
330: {
331: library_init(cfg, "test-runner");
332:
333: /* use non-blocking RNG to generate keys fast */
334: lib->settings->set_default_str(lib->settings,
335: "libstrongswan.plugins.random.random",
336: lib->settings->get_str(lib->settings,
337: "libstrongswan.plugins.random.urandom", "/dev/urandom"));
338: /* same for the gcrypt plugin */
339: lib->settings->set_default_str(lib->settings,
340: "libstrongswan.plugins.gcrypt.quick_random", "yes");
341:
342: if (lib->leak_detective)
343: {
344: /* disable leak reports during testing */
345: lib->leak_detective->set_report_cb(lib->leak_detective,
346: NULL, NULL, NULL);
347: }
348: if (init && !init(TRUE))
349: {
350: library_deinit();
351: return FALSE;
352: }
353: return TRUE;
354: }
355:
356: /**
357: * Failure description
358: */
359: typedef struct {
360: char *name;
361: char msg[4096 - sizeof(char*) - 2 * sizeof(int)];
362: const char *file;
363: int line;
364: int i;
365: backtrace_t *bt;
366: } failure_t;
367:
368: /**
369: * Data passed to leak report callbacks
370: */
371: typedef struct {
372: array_t *failures;
373: char *name;
374: int i;
375: int leaks;
376: } report_data_t;
377:
378: /**
379: * Leak report callback, build failures from leaks
380: */
381: static void report_leaks(report_data_t *data, int count, size_t bytes,
382: backtrace_t *bt, bool detailed)
383: {
384: failure_t failure = {
385: .name = data->name,
386: .i = data->i,
387: .bt = bt->clone(bt),
388: };
389:
390: snprintf(failure.msg, sizeof(failure.msg),
391: "Leak detected: %d allocations using %zu bytes", count, bytes);
392:
393: array_insert(data->failures, -1, &failure);
394: }
395:
396: /**
397: * Leak summary callback, check if any leaks found
398: */
399: static void sum_leaks(report_data_t *data, int count, size_t bytes,
400: int whitelisted)
401: {
402: data->leaks = count;
403: }
404:
405: /**
406: * Do library cleanup and optionally check for memory leaks
407: */
408: static bool post_test(test_runner_init_t init, bool check_leaks,
409: array_t *failures, char *name, int i, int *leaks)
410: {
411: report_data_t data = {
412: .failures = failures,
413: .name = name,
414: .i = i,
415: };
416:
417: if (init)
418: {
419: if (test_restore_point())
420: {
421: init(FALSE);
422: }
423: else
424: {
425: thread_cleanup_popall();
426: library_deinit();
427: return FALSE;
428: }
429: }
430: if (check_leaks && lib->leak_detective)
431: {
432: lib->leak_detective->set_report_cb(lib->leak_detective,
433: (leak_detective_report_cb_t)report_leaks,
434: (leak_detective_summary_cb_t)sum_leaks, &data);
435: }
436: library_deinit();
437:
438: *leaks = data.leaks;
439: return TRUE;
440: }
441:
442: /**
443: * Collect failure information, add failure_t to array
444: */
445: static void collect_failure_info(array_t *failures, char *name, int i)
446: {
447: failure_t failure = {
448: .name = name,
449: .i = i,
450: .bt = test_failure_backtrace(),
451: };
452:
453: failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
454: &failure.file);
455:
456: array_insert(failures, -1, &failure);
457: }
458:
459: /**
460: * Collect warning information, add failure_t to array
461: */
462: static bool collect_warning_info(array_t *warnings, char *name, int i)
463: {
464: failure_t warning = {
465: .name = name,
466: .i = i,
467: };
468:
469: warning.line = test_warning_get(warning.msg, sizeof(warning.msg),
470: &warning.file);
471: if (warning.line)
472: {
473: array_insert(warnings, -1, &warning);
474: }
475: return warning.line;
476: }
477:
478: /**
479: * Print array of collected failure_t to stderr
480: */
481: static void print_failures(array_t *failures, bool warnings)
482: {
483: failure_t failure;
484:
485: threads_init();
486: backtrace_init();
487:
488: while (array_remove(failures, 0, &failure))
489: {
490: if (warnings)
491: {
492: fprintf(stderr, " %sWarning in '%s': %s (",
493: TTY(YELLOW), failure.name, failure.msg);
494: }
495: else
496: {
497: fprintf(stderr, " %sFailure in '%s': %s (",
498: TTY(RED), failure.name, failure.msg);
499: }
500: if (failure.line)
501: {
502: fprintf(stderr, "%s:%d, ", failure.file, failure.line);
503: }
504: fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
505: if (failure.bt)
506: {
507: failure.bt->log(failure.bt, stderr, TRUE);
508: failure.bt->destroy(failure.bt);
509: }
510: }
511:
512: backtrace_deinit();
513: threads_deinit();
514: }
515:
1.1.1.2 ! misho 516: #if defined(CLOCK_THREAD_CPUTIME_ID) && defined(HAVE_CLOCK_GETTIME)
! 517:
! 518: /**
! 519: * Start a timer
! 520: */
! 521: static void start_timing(struct timespec *start)
! 522: {
! 523: clock_gettime(CLOCK_THREAD_CPUTIME_ID, start);
! 524: }
! 525:
! 526: /**
! 527: * End a timer, return ms
! 528: */
! 529: static double end_timing(struct timespec *start)
! 530: {
! 531: struct timespec end;
! 532:
! 533: if (!getenv("TESTS_TIMING"))
! 534: {
! 535: return 0;
! 536: }
! 537: clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
! 538: return (end.tv_nsec - start->tv_nsec) / 1000000.0 +
! 539: (end.tv_sec - start->tv_sec) * 1000.0;
! 540: }
! 541:
! 542: #else /* CLOCK_THREAD_CPUTIME_ID */
! 543:
! 544: #define start_timing(start) ((start)->tv_sec = 0, (start)->tv_nsec = 0)
! 545: #define end_timing(...) (0)
! 546:
! 547: #endif /* CLOCK_THREAD_CPUTIME_ID */
! 548:
1.1 misho 549: /**
550: * Run a single test case with fixtures
551: */
1.1.1.2 ! misho 552: static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg,
! 553: level_t level)
1.1 misho 554: {
555: enumerator_t *enumerator;
556: test_function_t *tfun;
1.1.1.2 ! misho 557: double *times;
! 558: double total_time = 0;
! 559: int tests = 0, ti = 0, passed = 0;
1.1 misho 560: array_t *failures, *warnings;
561:
1.1.1.2 ! misho 562: /* determine the number of tests we will run */
! 563: enumerator = array_create_enumerator(tcase->functions);
! 564: while (enumerator->enumerate(enumerator, &tfun))
! 565: {
! 566: tests += tfun->end - tfun->start;
! 567: }
! 568: enumerator->destroy(enumerator);
! 569: times = calloc(tests, sizeof(double));
! 570:
1.1 misho 571: failures = array_create(sizeof(failure_t), 0);
572: warnings = array_create(sizeof(failure_t), 0);
573:
574: fprintf(stderr, " Running case '%s': ", tcase->name);
575: fflush(stderr);
576:
577: enumerator = array_create_enumerator(tcase->functions);
578: while (enumerator->enumerate(enumerator, &tfun))
579: {
580: int i, rounds = 0;
581:
582: for (i = tfun->start; i < tfun->end; i++)
583: {
584: if (pre_test(init, cfg))
585: {
1.1.1.2 ! misho 586: struct timespec start;
1.1 misho 587: bool ok = FALSE;
588: int leaks = 0;
589:
1.1.1.2 ! misho 590: if (level >= 0)
! 591: {
! 592: fprintf(stderr, "\nRunning function '%s' [%d]:\n",
! 593: tfun->name, i);
! 594: }
! 595:
1.1 misho 596: test_setup_timeout(tcase->timeout);
1.1.1.2 ! misho 597: start_timing(&start);
1.1 misho 598:
1.1.1.2 ! misho 599: if (call_fixture(tcase, TRUE, i))
1.1 misho 600: {
601: if (run_test(tfun, i))
602: {
1.1.1.2 ! misho 603: if (call_fixture(tcase, FALSE, i))
1.1 misho 604: {
605: ok = TRUE;
606: }
607: }
608: else
609: {
1.1.1.2 ! misho 610: call_fixture(tcase, FALSE, i);
1.1 misho 611: }
612: }
613: if (!post_test(init, ok, failures, tfun->name, i, &leaks))
614: {
615: ok = FALSE;
616: }
617:
1.1.1.2 ! misho 618: times[ti] = end_timing(&start);
! 619: total_time += times[ti++];
1.1 misho 620: test_setup_timeout(0);
621:
622: if (ok)
623: {
624: if (!leaks)
625: {
626: rounds++;
627: if (!collect_warning_info(warnings, tfun->name, i))
628: {
629: fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
630: }
631: else
632: {
633: fprintf(stderr, "%s~%s", TTY(YELLOW), TTY(DEF));
634: }
635: }
636: }
637: else
638: {
639: collect_failure_info(failures, tfun->name, i);
640: }
641: if (!ok || leaks)
642: {
643: fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
644: }
645: }
646: else
647: {
648: fprintf(stderr, "!");
649: }
650: }
651: fflush(stderr);
652: if (rounds == tfun->end - tfun->start)
653: {
654: passed++;
655: }
656: }
657: enumerator->destroy(enumerator);
658:
1.1.1.2 ! misho 659: if (total_time)
! 660: {
! 661: fprintf(stderr, " %s%s%.3f ms%s", tty_escape_get(2, TTY_BOLD),
! 662: TTY(BLUE), total_time, tty_escape_get(2, TTY_RESET));
! 663: if (ti > 1)
! 664: {
! 665: fprintf(stderr, " %s[", TTY(BLUE));
! 666: for (ti = 0; ti < tests; ti++)
! 667: {
! 668: fprintf(stderr, "%s%.3f ms", times[ti], ti == 0 ? "" : ", ");
! 669: }
! 670: fprintf(stderr, "]%s", TTY(DEF));
! 671: }
! 672: }
1.1 misho 673: fprintf(stderr, "\n");
674:
675: print_failures(warnings, TRUE);
676: print_failures(failures, FALSE);
677: array_destroy(failures);
678: array_destroy(warnings);
679:
680: return passed == array_count(tcase->functions);
681: }
682:
683: /**
684: * Run a single test suite
685: */
1.1.1.2 ! misho 686: static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg,
! 687: level_t level)
1.1 misho 688: {
689: enumerator_t *enumerator;
690: test_case_t *tcase;
691: int passed = 0;
692:
693: fprintf(stderr, " Running suite '%s':\n", suite->name);
694:
695: enumerator = array_create_enumerator(suite->tcases);
696: while (enumerator->enumerate(enumerator, &tcase))
697: {
1.1.1.2 ! misho 698: if (run_case(tcase, init, cfg, level))
1.1 misho 699: {
700: passed++;
701: }
702: }
703: enumerator->destroy(enumerator);
704:
705: if (passed == array_count(suite->tcases))
706: {
707: fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
708: TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
709: return TRUE;
710: }
711: fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
712: TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
713: return FALSE;
714: }
715:
716: /**
717: * See header.
718: */
719: int test_runner_run(const char *name, test_configuration_t configs[],
720: test_runner_init_t init)
721: {
722: array_t *suites;
723: test_suite_t *suite;
724: enumerator_t *enumerator;
725: int passed = 0, result;
726: level_t level = LEVEL_SILENT;
727: char *cfg, *runners, *verbosity;
728:
729: /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
730: dup2(2, 1);
731:
732: runners = getenv("TESTS_RUNNERS");
733: if (runners && !is_in_filter(name, runners))
734: {
735: return EXIT_SUCCESS;
736: }
737:
738: cfg = getenv("TESTS_STRONGSWAN_CONF");
739:
740: suites = load_suites(configs, init, cfg);
741: if (!suites)
742: {
743: return EXIT_FAILURE;
744: }
745:
746: verbosity = getenv("TESTS_VERBOSITY");
747: if (verbosity)
748: {
749: level = atoi(verbosity);
750: }
751: dbg_default_set_level(level);
752:
753: fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
754:
755: enumerator = array_create_enumerator(suites);
756: while (enumerator->enumerate(enumerator, &suite))
757: {
1.1.1.2 ! misho 758: if (run_suite(suite, init, cfg, level))
1.1 misho 759: {
760: passed++;
761: }
762: }
763: enumerator->destroy(enumerator);
764:
765: if (passed == array_count(suites))
766: {
767: fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
768: TTY(GREEN), array_count(suites), name, TTY(DEF));
769: result = EXIT_SUCCESS;
770: }
771: else
772: {
773: fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
774: TTY(RED), passed, array_count(suites), name, TTY(DEF));
775: result = EXIT_FAILURE;
776: }
777:
778: unload_suites(suites);
779:
780: return result;
781: }