Annotation of embedaddon/strongswan/src/libstrongswan/tests/test_runner.c, revision 1.1
1.1 ! misho 1: /*
! 2: * Copyright (C) 2013 Tobias Brunner
! 3: * HSR Hochschule fuer Technik Rapperswil
! 4: * Copyright (C) 2013 Martin Willi
! 5: * Copyright (C) 2013 revosec AG
! 6: *
! 7: * This program is free software; you can redistribute it and/or modify it
! 8: * under the terms of the GNU General Public License as published by the
! 9: * Free Software Foundation; either version 2 of the License, or (at your
! 10: * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
! 11: *
! 12: * This program is distributed in the hope that it will be useful, but
! 13: * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
! 14: * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
! 15: * for more details.
! 16: */
! 17:
! 18: #include "test_runner.h"
! 19:
! 20: #include <library.h>
! 21: #include <threading/thread.h>
! 22: #include <plugins/plugin_feature.h>
! 23: #include <collections/array.h>
! 24: #include <utils/test.h>
! 25:
! 26: #include <stdlib.h>
! 27: #include <dirent.h>
! 28: #include <unistd.h>
! 29: #include <limits.h>
! 30: #include <stdlib.h>
! 31:
! 32: /**
! 33: * Get a tty color escape character for stderr
! 34: */
! 35: #define TTY(color) tty_escape_get(2, TTY_FG_##color)
! 36:
! 37: /**
! 38: * A global symbol indicating libtest linkage
! 39: */
! 40: #ifdef WIN32
! 41: __declspec(dllexport)
! 42: #endif
! 43: bool test_runner_available = TRUE;
! 44:
! 45: /**
! 46: * Destroy a single test suite and associated data
! 47: */
! 48: static void destroy_suite(test_suite_t *suite)
! 49: {
! 50: test_case_t *tcase;
! 51:
! 52: while (array_remove(suite->tcases, 0, &tcase))
! 53: {
! 54: array_destroy(tcase->functions);
! 55: array_destroy(tcase->fixtures);
! 56: }
! 57: free(suite);
! 58: }
! 59:
! 60: /**
! 61: * Filter loaded test suites, either remove suites listed (exclude=TRUE), or all
! 62: * that are not listed (exclude=FALSE).
! 63: */
! 64: static void apply_filter(array_t *loaded, char *filter, bool exclude)
! 65: {
! 66: enumerator_t *enumerator, *names;
! 67: hashtable_t *listed;
! 68: test_suite_t *suite;
! 69: char *name;
! 70:
! 71: listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
! 72: names = enumerator_create_token(filter, ",", " ");
! 73: while (names->enumerate(names, &name))
! 74: {
! 75: listed->put(listed, name, name);
! 76: }
! 77: enumerator = array_create_enumerator(loaded);
! 78: while (enumerator->enumerate(enumerator, &suite))
! 79: {
! 80: if ((exclude && listed->get(listed, suite->name)) ||
! 81: (!exclude && !listed->get(listed, suite->name)))
! 82: {
! 83: array_remove_at(loaded, enumerator);
! 84: destroy_suite(suite);
! 85: }
! 86: }
! 87: enumerator->destroy(enumerator);
! 88: listed->destroy(listed);
! 89: names->destroy(names);
! 90: }
! 91:
! 92: /**
! 93: * Check if the given string is contained in the filter string.
! 94: */
! 95: static bool is_in_filter(const char *find, char *filter)
! 96: {
! 97: enumerator_t *names;
! 98: bool found = FALSE;
! 99: char *name;
! 100:
! 101: names = enumerator_create_token(filter, ",", " ");
! 102: while (names->enumerate(names, &name))
! 103: {
! 104: if (streq(name, find))
! 105: {
! 106: found = TRUE;
! 107: break;
! 108: }
! 109: }
! 110: names->destroy(names);
! 111: return found;
! 112: }
! 113:
! 114: /**
! 115: * Removes and destroys test suites that are not selected or
! 116: * explicitly excluded.
! 117: */
! 118: static void filter_suites(array_t *loaded)
! 119: {
! 120: char *filter;
! 121:
! 122: filter = getenv("TESTS_SUITES");
! 123: if (filter)
! 124: {
! 125: apply_filter(loaded, filter, FALSE);
! 126: }
! 127: filter = getenv("TESTS_SUITES_EXCLUDE");
! 128: if (filter)
! 129: {
! 130: apply_filter(loaded, filter, TRUE);
! 131: }
! 132: }
! 133:
! 134: /**
! 135: * Load all available test suites, or optionally only selected ones.
! 136: */
! 137: static array_t *load_suites(test_configuration_t configs[],
! 138: test_runner_init_t init, char *cfg)
! 139: {
! 140: array_t *suites;
! 141: bool old = FALSE;
! 142: int i;
! 143:
! 144: library_init(cfg, "test-runner");
! 145:
! 146: test_setup_handler();
! 147:
! 148: if (init && !init(TRUE))
! 149: {
! 150: library_deinit();
! 151: return NULL;
! 152: }
! 153: lib->plugins->status(lib->plugins, LEVEL_CTRL);
! 154:
! 155: if (lib->leak_detective)
! 156: {
! 157: old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
! 158: }
! 159:
! 160: suites = array_create(0, 0);
! 161:
! 162: for (i = 0; configs[i].suite; i++)
! 163: {
! 164: if (configs[i].feature.type == 0 ||
! 165: lib->plugins->has_feature(lib->plugins, configs[i].feature))
! 166: {
! 167: array_insert(suites, -1, configs[i].suite());
! 168: }
! 169: }
! 170: filter_suites(suites);
! 171:
! 172: if (lib->leak_detective)
! 173: {
! 174: lib->leak_detective->set_state(lib->leak_detective, old);
! 175: }
! 176:
! 177: if (init)
! 178: {
! 179: init(FALSE);
! 180: }
! 181: library_deinit();
! 182:
! 183: return suites;
! 184: }
! 185:
! 186: /**
! 187: * Unload and destroy test suites and associated data
! 188: */
! 189: static void unload_suites(array_t *suites)
! 190: {
! 191: test_suite_t *suite;
! 192:
! 193: while (array_remove(suites, 0, &suite))
! 194: {
! 195: destroy_suite(suite);
! 196: }
! 197: array_destroy(suites);
! 198: }
! 199:
! 200: /**
! 201: * Run a single test function, return FALSE on failure
! 202: */
! 203: static bool run_test(test_function_t *tfun, int i)
! 204: {
! 205: if (test_restore_point())
! 206: {
! 207: tfun->cb(i);
! 208: return TRUE;
! 209: }
! 210: thread_cleanup_popall();
! 211: return FALSE;
! 212: }
! 213:
! 214: /**
! 215: * Invoke fixture setup/teardown
! 216: */
! 217: static bool call_fixture(test_case_t *tcase, bool up)
! 218: {
! 219: enumerator_t *enumerator;
! 220: test_fixture_t *fixture;
! 221: bool failure = FALSE;
! 222:
! 223: enumerator = array_create_enumerator(tcase->fixtures);
! 224: while (enumerator->enumerate(enumerator, &fixture))
! 225: {
! 226: if (test_restore_point())
! 227: {
! 228: if (up)
! 229: {
! 230: if (fixture->setup)
! 231: {
! 232: fixture->setup();
! 233: }
! 234: }
! 235: else
! 236: {
! 237: if (fixture->teardown)
! 238: {
! 239: fixture->teardown();
! 240: }
! 241: }
! 242: }
! 243: else
! 244: {
! 245: thread_cleanup_popall();
! 246: failure = TRUE;
! 247: break;
! 248: }
! 249: }
! 250: enumerator->destroy(enumerator);
! 251:
! 252: return !failure;
! 253: }
! 254:
! 255: /**
! 256: * Test initialization, initializes libstrongswan for the next run
! 257: */
! 258: static bool pre_test(test_runner_init_t init, char *cfg)
! 259: {
! 260: library_init(cfg, "test-runner");
! 261:
! 262: /* use non-blocking RNG to generate keys fast */
! 263: lib->settings->set_default_str(lib->settings,
! 264: "libstrongswan.plugins.random.random",
! 265: lib->settings->get_str(lib->settings,
! 266: "libstrongswan.plugins.random.urandom", "/dev/urandom"));
! 267: /* same for the gcrypt plugin */
! 268: lib->settings->set_default_str(lib->settings,
! 269: "libstrongswan.plugins.gcrypt.quick_random", "yes");
! 270:
! 271: if (lib->leak_detective)
! 272: {
! 273: /* disable leak reports during testing */
! 274: lib->leak_detective->set_report_cb(lib->leak_detective,
! 275: NULL, NULL, NULL);
! 276: }
! 277: if (init && !init(TRUE))
! 278: {
! 279: library_deinit();
! 280: return FALSE;
! 281: }
! 282: return TRUE;
! 283: }
! 284:
! 285: /**
! 286: * Failure description
! 287: */
! 288: typedef struct {
! 289: char *name;
! 290: char msg[4096 - sizeof(char*) - 2 * sizeof(int)];
! 291: const char *file;
! 292: int line;
! 293: int i;
! 294: backtrace_t *bt;
! 295: } failure_t;
! 296:
! 297: /**
! 298: * Data passed to leak report callbacks
! 299: */
! 300: typedef struct {
! 301: array_t *failures;
! 302: char *name;
! 303: int i;
! 304: int leaks;
! 305: } report_data_t;
! 306:
! 307: /**
! 308: * Leak report callback, build failures from leaks
! 309: */
! 310: static void report_leaks(report_data_t *data, int count, size_t bytes,
! 311: backtrace_t *bt, bool detailed)
! 312: {
! 313: failure_t failure = {
! 314: .name = data->name,
! 315: .i = data->i,
! 316: .bt = bt->clone(bt),
! 317: };
! 318:
! 319: snprintf(failure.msg, sizeof(failure.msg),
! 320: "Leak detected: %d allocations using %zu bytes", count, bytes);
! 321:
! 322: array_insert(data->failures, -1, &failure);
! 323: }
! 324:
! 325: /**
! 326: * Leak summary callback, check if any leaks found
! 327: */
! 328: static void sum_leaks(report_data_t *data, int count, size_t bytes,
! 329: int whitelisted)
! 330: {
! 331: data->leaks = count;
! 332: }
! 333:
! 334: /**
! 335: * Do library cleanup and optionally check for memory leaks
! 336: */
! 337: static bool post_test(test_runner_init_t init, bool check_leaks,
! 338: array_t *failures, char *name, int i, int *leaks)
! 339: {
! 340: report_data_t data = {
! 341: .failures = failures,
! 342: .name = name,
! 343: .i = i,
! 344: };
! 345:
! 346: if (init)
! 347: {
! 348: if (test_restore_point())
! 349: {
! 350: init(FALSE);
! 351: }
! 352: else
! 353: {
! 354: thread_cleanup_popall();
! 355: library_deinit();
! 356: return FALSE;
! 357: }
! 358: }
! 359: if (check_leaks && lib->leak_detective)
! 360: {
! 361: lib->leak_detective->set_report_cb(lib->leak_detective,
! 362: (leak_detective_report_cb_t)report_leaks,
! 363: (leak_detective_summary_cb_t)sum_leaks, &data);
! 364: }
! 365: library_deinit();
! 366:
! 367: *leaks = data.leaks;
! 368: return TRUE;
! 369: }
! 370:
! 371: /**
! 372: * Collect failure information, add failure_t to array
! 373: */
! 374: static void collect_failure_info(array_t *failures, char *name, int i)
! 375: {
! 376: failure_t failure = {
! 377: .name = name,
! 378: .i = i,
! 379: .bt = test_failure_backtrace(),
! 380: };
! 381:
! 382: failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
! 383: &failure.file);
! 384:
! 385: array_insert(failures, -1, &failure);
! 386: }
! 387:
! 388: /**
! 389: * Collect warning information, add failure_t to array
! 390: */
! 391: static bool collect_warning_info(array_t *warnings, char *name, int i)
! 392: {
! 393: failure_t warning = {
! 394: .name = name,
! 395: .i = i,
! 396: };
! 397:
! 398: warning.line = test_warning_get(warning.msg, sizeof(warning.msg),
! 399: &warning.file);
! 400: if (warning.line)
! 401: {
! 402: array_insert(warnings, -1, &warning);
! 403: }
! 404: return warning.line;
! 405: }
! 406:
! 407: /**
! 408: * Print array of collected failure_t to stderr
! 409: */
! 410: static void print_failures(array_t *failures, bool warnings)
! 411: {
! 412: failure_t failure;
! 413:
! 414: threads_init();
! 415: backtrace_init();
! 416:
! 417: while (array_remove(failures, 0, &failure))
! 418: {
! 419: if (warnings)
! 420: {
! 421: fprintf(stderr, " %sWarning in '%s': %s (",
! 422: TTY(YELLOW), failure.name, failure.msg);
! 423: }
! 424: else
! 425: {
! 426: fprintf(stderr, " %sFailure in '%s': %s (",
! 427: TTY(RED), failure.name, failure.msg);
! 428: }
! 429: if (failure.line)
! 430: {
! 431: fprintf(stderr, "%s:%d, ", failure.file, failure.line);
! 432: }
! 433: fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
! 434: if (failure.bt)
! 435: {
! 436: failure.bt->log(failure.bt, stderr, TRUE);
! 437: failure.bt->destroy(failure.bt);
! 438: }
! 439: }
! 440:
! 441: backtrace_deinit();
! 442: threads_deinit();
! 443: }
! 444:
! 445: /**
! 446: * Run a single test case with fixtures
! 447: */
! 448: static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
! 449: {
! 450: enumerator_t *enumerator;
! 451: test_function_t *tfun;
! 452: int passed = 0;
! 453: array_t *failures, *warnings;
! 454:
! 455: failures = array_create(sizeof(failure_t), 0);
! 456: warnings = array_create(sizeof(failure_t), 0);
! 457:
! 458: fprintf(stderr, " Running case '%s': ", tcase->name);
! 459: fflush(stderr);
! 460:
! 461: enumerator = array_create_enumerator(tcase->functions);
! 462: while (enumerator->enumerate(enumerator, &tfun))
! 463: {
! 464: int i, rounds = 0;
! 465:
! 466: for (i = tfun->start; i < tfun->end; i++)
! 467: {
! 468: if (pre_test(init, cfg))
! 469: {
! 470: bool ok = FALSE;
! 471: int leaks = 0;
! 472:
! 473: test_setup_timeout(tcase->timeout);
! 474:
! 475: if (call_fixture(tcase, TRUE))
! 476: {
! 477: if (run_test(tfun, i))
! 478: {
! 479: if (call_fixture(tcase, FALSE))
! 480: {
! 481: ok = TRUE;
! 482: }
! 483: }
! 484: else
! 485: {
! 486: call_fixture(tcase, FALSE);
! 487: }
! 488: }
! 489: if (!post_test(init, ok, failures, tfun->name, i, &leaks))
! 490: {
! 491: ok = FALSE;
! 492: }
! 493:
! 494: test_setup_timeout(0);
! 495:
! 496: if (ok)
! 497: {
! 498: if (!leaks)
! 499: {
! 500: rounds++;
! 501: if (!collect_warning_info(warnings, tfun->name, i))
! 502: {
! 503: fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
! 504: }
! 505: else
! 506: {
! 507: fprintf(stderr, "%s~%s", TTY(YELLOW), TTY(DEF));
! 508: }
! 509: }
! 510: }
! 511: else
! 512: {
! 513: collect_failure_info(failures, tfun->name, i);
! 514: }
! 515: if (!ok || leaks)
! 516: {
! 517: fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
! 518: }
! 519: }
! 520: else
! 521: {
! 522: fprintf(stderr, "!");
! 523: }
! 524: }
! 525: fflush(stderr);
! 526: if (rounds == tfun->end - tfun->start)
! 527: {
! 528: passed++;
! 529: }
! 530: }
! 531: enumerator->destroy(enumerator);
! 532:
! 533: fprintf(stderr, "\n");
! 534:
! 535: print_failures(warnings, TRUE);
! 536: print_failures(failures, FALSE);
! 537: array_destroy(failures);
! 538: array_destroy(warnings);
! 539:
! 540: return passed == array_count(tcase->functions);
! 541: }
! 542:
! 543: /**
! 544: * Run a single test suite
! 545: */
! 546: static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
! 547: {
! 548: enumerator_t *enumerator;
! 549: test_case_t *tcase;
! 550: int passed = 0;
! 551:
! 552: fprintf(stderr, " Running suite '%s':\n", suite->name);
! 553:
! 554: enumerator = array_create_enumerator(suite->tcases);
! 555: while (enumerator->enumerate(enumerator, &tcase))
! 556: {
! 557: if (run_case(tcase, init, cfg))
! 558: {
! 559: passed++;
! 560: }
! 561: }
! 562: enumerator->destroy(enumerator);
! 563:
! 564: if (passed == array_count(suite->tcases))
! 565: {
! 566: fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
! 567: TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
! 568: return TRUE;
! 569: }
! 570: fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
! 571: TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
! 572: return FALSE;
! 573: }
! 574:
! 575: /**
! 576: * See header.
! 577: */
! 578: int test_runner_run(const char *name, test_configuration_t configs[],
! 579: test_runner_init_t init)
! 580: {
! 581: array_t *suites;
! 582: test_suite_t *suite;
! 583: enumerator_t *enumerator;
! 584: int passed = 0, result;
! 585: level_t level = LEVEL_SILENT;
! 586: char *cfg, *runners, *verbosity;
! 587:
! 588: /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
! 589: dup2(2, 1);
! 590:
! 591: runners = getenv("TESTS_RUNNERS");
! 592: if (runners && !is_in_filter(name, runners))
! 593: {
! 594: return EXIT_SUCCESS;
! 595: }
! 596:
! 597: cfg = getenv("TESTS_STRONGSWAN_CONF");
! 598:
! 599: suites = load_suites(configs, init, cfg);
! 600: if (!suites)
! 601: {
! 602: return EXIT_FAILURE;
! 603: }
! 604:
! 605: verbosity = getenv("TESTS_VERBOSITY");
! 606: if (verbosity)
! 607: {
! 608: level = atoi(verbosity);
! 609: }
! 610: dbg_default_set_level(level);
! 611:
! 612: fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
! 613:
! 614: enumerator = array_create_enumerator(suites);
! 615: while (enumerator->enumerate(enumerator, &suite))
! 616: {
! 617: if (run_suite(suite, init, cfg))
! 618: {
! 619: passed++;
! 620: }
! 621: }
! 622: enumerator->destroy(enumerator);
! 623:
! 624: if (passed == array_count(suites))
! 625: {
! 626: fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
! 627: TTY(GREEN), array_count(suites), name, TTY(DEF));
! 628: result = EXIT_SUCCESS;
! 629: }
! 630: else
! 631: {
! 632: fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
! 633: TTY(RED), passed, array_count(suites), name, TTY(DEF));
! 634: result = EXIT_FAILURE;
! 635: }
! 636:
! 637: unload_suites(suites);
! 638:
! 639: return result;
! 640: }
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to test_runner.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / strongswan / src / libstrongswan / tests