static cairo_test_status_t cairo_test_expecting (cairo_test_t *test, cairo_test_status_t expectation) { /* we use volatile here to make sure values are not clobbered * by longjmp */ volatile size_t i, j, num_targets; volatile cairo_bool_t limited_targets = FALSE, print_fail_on_stdout = TRUE; #ifdef HAVE_SIGNAL_H void (*old_segfault_handler)(int); #endif volatile cairo_test_status_t status, ret; cairo_boilerplate_target_t ** volatile targets_to_test; #ifdef HAVE_UNISTD_H if (isatty (2)) { fail_face = "\033[41m\033[37m\033[1m"; normal_face = "\033[m"; if (isatty (1)) print_fail_on_stdout = FALSE; } #endif srcdir = getenv ("srcdir"); if (!srcdir) srcdir = "."; cairo_test_init (test->name); printf ("%s\n", test->description); if (expectation == CAIRO_TEST_FAILURE) printf ("Expecting failure\n"); { int tmp_num_targets; cairo_bool_t tmp_limited_targets; targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets); num_targets = tmp_num_targets; limited_targets = tmp_limited_targets; } /* The intended logic here is that we return overall SUCCESS * iff. there is at least one tested backend and that all tested * backends return SUCCESS, OR, there's backends were manually * limited, and none were tested. * In other words: * * if backends limited and no backend tested * -> SUCCESS * else if any backend not SUCCESS * -> FAILURE * else if all backends UNTESTED * -> FAILURE * else (== some backend SUCCESS) * -> SUCCESS * * Also, on a crash, run no further tests. */ status = ret = CAIRO_TEST_UNTESTED; for (i = 0; i < num_targets && status != CAIRO_TEST_CRASHED; i++) { for (j = 0; j < NUM_DEVICE_OFFSETS; j++) { cairo_boilerplate_target_t * volatile target = targets_to_test[i]; volatile int dev_offset = j * 25; cairo_test_log ("Testing %s with %s target (dev offset %d)\n", test->name, target->name, dev_offset); printf ("%s-%s-%s [%d]:\t", test->name, target->name, cairo_boilerplate_content_name (target->content), dev_offset); #ifdef HAVE_SIGNAL_H /* Set up a checkpoint to get back to in case of segfaults. */ old_segfault_handler = signal (SIGSEGV, segfault_handler); if (0 == setjmp (jmpbuf)) #endif status = cairo_test_for_target (test, target, dev_offset); #ifdef HAVE_SIGNAL_H else status = CAIRO_TEST_CRASHED; signal (SIGSEGV, old_segfault_handler); #endif cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d RESULT: ", test->name, target->name, cairo_boilerplate_content_name (target->content), dev_offset); switch (status) { case CAIRO_TEST_SUCCESS: printf ("PASS\n"); cairo_test_log ("PASS\n"); if (ret == CAIRO_TEST_UNTESTED) ret = CAIRO_TEST_SUCCESS; break; case CAIRO_TEST_UNTESTED: printf ("UNTESTED\n"); cairo_test_log ("UNTESTED\n"); break; case CAIRO_TEST_CRASHED: if (print_fail_on_stdout) { printf ("!!!CRASHED!!!\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } cairo_test_log ("CRASHED\n"); fprintf (stderr, "%s-%s-%s [%d]:\t%s!!!CRASHED!!!%s\n", test->name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, fail_face, normal_face); ret = CAIRO_TEST_FAILURE; break; default: case CAIRO_TEST_FAILURE: if (expectation == CAIRO_TEST_FAILURE) { printf ("XFAIL\n"); cairo_test_log ("XFAIL\n"); } else { if (print_fail_on_stdout) { printf ("FAIL\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } fprintf (stderr, "%s-%s-%s [%d]:\t%sFAIL%s\n", test->name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, fail_face, normal_face); cairo_test_log ("FAIL\n"); } ret = status; break; } } } if (ret != CAIRO_TEST_SUCCESS) printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX); /* if the set of targets to test was limited using CAIRO_TEST_TARGET, we * behave slightly differently, to ensure that limiting the targets does * not increase the number of tests failing. */ if (limited_targets) { /* if all untested, success */ if (ret == CAIRO_TEST_UNTESTED) { printf ("None of the tested backends passed, but tested targets are manually limited.\n" "Passing the test, to not fail the suite.\n"); ret = CAIRO_TEST_SUCCESS; } /* if all passed, but expecting failure, return failure to not * trigger an XPASS failure */ if (expectation == CAIRO_TEST_FAILURE && ret == CAIRO_TEST_SUCCESS) { printf ("All tested backends passed, but tested targets are manually limited\n" "and the test suite expects this test to fail for at least one target.\n" "Intentionally failing the test, to not fail the suite.\n"); ret = CAIRO_TEST_FAILURE; } } else { if (ret == CAIRO_TEST_UNTESTED) ret = CAIRO_TEST_FAILURE; } cairo_test_fini (); cairo_boilerplate_free_targets (targets_to_test); return ret; }
cairo_test_status_t _cairo_test_context_run_for_target (cairo_test_context_t *ctx, const cairo_boilerplate_target_t *target, cairo_bool_t similar, int dev_offset) { cairo_test_status_t status; if (target->get_image_surface == NULL) return CAIRO_TEST_UNTESTED; if (similar && ! cairo_test_target_has_similar (ctx, target)) return CAIRO_TEST_UNTESTED; cairo_test_log (ctx, "Testing %s with %s%s target (dev offset %d)\n", ctx->test_name, similar ? " (similar) " : "", target->name, dev_offset); printf ("%s.%s.%s [%d]%s:\t", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)": ""); fflush (stdout); #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H) if (! RUNNING_ON_VALGRIND) { void (* volatile old_segfault_handler)(int); void (* volatile old_segfpe_handler)(int); void (* volatile old_sigpipe_handler)(int); void (* volatile old_sigabrt_handler)(int); void (* volatile old_sigalrm_handler)(int); /* Set up a checkpoint to get back to in case of segfaults. */ #ifdef SIGSEGV old_segfault_handler = signal (SIGSEGV, segfault_handler); #endif #ifdef SIGFPE old_segfpe_handler = signal (SIGFPE, segfault_handler); #endif #ifdef SIGPIPE old_sigpipe_handler = signal (SIGPIPE, segfault_handler); #endif #ifdef SIGABRT old_sigabrt_handler = signal (SIGABRT, segfault_handler); #endif #ifdef SIGALRM old_sigalrm_handler = signal (SIGALRM, segfault_handler); #endif if (0 == setjmp (jmpbuf)) status = cairo_test_for_target (ctx, target, dev_offset, similar); else status = CAIRO_TEST_CRASHED; #ifdef SIGSEGV signal (SIGSEGV, old_segfault_handler); #endif #ifdef SIGFPE signal (SIGFPE, old_segfpe_handler); #endif #ifdef SIGPIPE signal (SIGPIPE, old_sigpipe_handler); #endif #ifdef SIGABRT signal (SIGABRT, old_sigabrt_handler); #endif #ifdef SIGALRM signal (SIGALRM, old_sigalrm_handler); #endif } else { status = cairo_test_for_target (ctx, target, dev_offset, similar); } #else status = cairo_test_for_target (ctx, target, dev_offset, similar); #endif cairo_test_log (ctx, "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar); switch (status) { case CAIRO_TEST_SUCCESS: printf ("PASS\n"); cairo_test_log (ctx, "PASS\n"); break; case CAIRO_TEST_UNTESTED: printf ("UNTESTED\n"); cairo_test_log (ctx, "UNTESTED\n"); break; default: case CAIRO_TEST_CRASHED: if (print_fail_on_stdout) { printf ("!!!CRASHED!!!\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } cairo_test_log (ctx, "CRASHED\n"); fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!CRASHED!!!%s\n", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", fail_face, normal_face); break; case CAIRO_TEST_ERROR: if (print_fail_on_stdout) { printf ("!!!ERROR!!!\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } cairo_test_log (ctx, "ERROR\n"); fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", fail_face, normal_face); break; case CAIRO_TEST_XFAILURE: if (print_fail_on_stdout) { printf ("XFAIL\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", xfail_face, normal_face); cairo_test_log (ctx, "XFAIL\n"); break; case CAIRO_TEST_NEW: if (print_fail_on_stdout) { printf ("NEW\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } fprintf (stderr, "%s.%s.%s [%d]%s:\t%sNEW%s\n", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", fail_face, normal_face); cairo_test_log (ctx, "NEW\n"); break; case CAIRO_TEST_NO_MEMORY: case CAIRO_TEST_FAILURE: if (print_fail_on_stdout) { printf ("FAIL\n"); } else { /* eat the test name */ printf ("\r"); fflush (stdout); } fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n", ctx->test_name, target->name, cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", fail_face, normal_face); cairo_test_log (ctx, "FAIL\n"); break; } fflush (stdout); return status; }