static cairo_test_status_t
cairo_test_expecting (cairo_test_t *test,
cairo_test_status_t expectation)
{
/* we use volatile here to make sure values are not clobbered
* by longjmp */
volatile size_t i, j, num_targets;
volatile cairo_bool_t limited_targets = FALSE, print_fail_on_stdout = TRUE;
#ifdef HAVE_SIGNAL_H
void (*old_segfault_handler)(int);
#endif
volatile cairo_test_status_t status, ret;
cairo_boilerplate_target_t ** volatile targets_to_test;
#ifdef HAVE_UNISTD_H
if (isatty (2)) {
fail_face = "\033[41m\033[37m\033[1m";
normal_face = "\033[m";
if (isatty (1))
print_fail_on_stdout = FALSE;
}
#endif
srcdir = getenv ("srcdir");
if (!srcdir)
srcdir = ".";
cairo_test_init (test->name);
printf ("%s\n", test->description);
if (expectation == CAIRO_TEST_FAILURE)
printf ("Expecting failure\n");
{
int tmp_num_targets;
cairo_bool_t tmp_limited_targets;
targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
num_targets = tmp_num_targets;
limited_targets = tmp_limited_targets;
}
/* The intended logic here is that we return overall SUCCESS
* iff. there is at least one tested backend and that all tested
* backends return SUCCESS, OR, there's backends were manually
* limited, and none were tested.
* In other words:
*
* if backends limited and no backend tested
* -> SUCCESS
* else if any backend not SUCCESS
* -> FAILURE
* else if all backends UNTESTED
* -> FAILURE
* else (== some backend SUCCESS)
* -> SUCCESS
*
* Also, on a crash, run no further tests.
*/
status = ret = CAIRO_TEST_UNTESTED;
for (i = 0; i < num_targets && status != CAIRO_TEST_CRASHED; i++) {
for (j = 0; j < NUM_DEVICE_OFFSETS; j++) {
cairo_boilerplate_target_t * volatile target = targets_to_test[i];
volatile int dev_offset = j * 25;
cairo_test_log ("Testing %s with %s target (dev offset %d)\n", test->name, target->name, dev_offset);
printf ("%s-%s-%s [%d]:\t", test->name, target->name,
cairo_boilerplate_content_name (target->content),
dev_offset);
#ifdef HAVE_SIGNAL_H
/* Set up a checkpoint to get back to in case of segfaults. */
old_segfault_handler = signal (SIGSEGV, segfault_handler);
if (0 == setjmp (jmpbuf))
#endif
status = cairo_test_for_target (test, target, dev_offset);
#ifdef HAVE_SIGNAL_H
else
status = CAIRO_TEST_CRASHED;
signal (SIGSEGV, old_segfault_handler);
#endif
cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d RESULT: ",
test->name, target->name,
cairo_boilerplate_content_name (target->content),
dev_offset);
switch (status) {
case CAIRO_TEST_SUCCESS:
printf ("PASS\n");
cairo_test_log ("PASS\n");
if (ret == CAIRO_TEST_UNTESTED)
ret = CAIRO_TEST_SUCCESS;
break;
case CAIRO_TEST_UNTESTED:
printf ("UNTESTED\n");
cairo_test_log ("UNTESTED\n");
break;
case CAIRO_TEST_CRASHED:
if (print_fail_on_stdout) {
printf ("!!!CRASHED!!!\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
cairo_test_log ("CRASHED\n");
fprintf (stderr, "%s-%s-%s [%d]:\t%s!!!CRASHED!!!%s\n",
test->name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset,
fail_face, normal_face);
ret = CAIRO_TEST_FAILURE;
break;
default:
case CAIRO_TEST_FAILURE:
if (expectation == CAIRO_TEST_FAILURE) {
printf ("XFAIL\n");
cairo_test_log ("XFAIL\n");
} else {
if (print_fail_on_stdout) {
printf ("FAIL\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
fprintf (stderr, "%s-%s-%s [%d]:\t%sFAIL%s\n",
test->name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset,
fail_face, normal_face);
cairo_test_log ("FAIL\n");
}
ret = status;
break;
}
}
}
if (ret != CAIRO_TEST_SUCCESS)
printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX);
/* if the set of targets to test was limited using CAIRO_TEST_TARGET, we
* behave slightly differently, to ensure that limiting the targets does
* not increase the number of tests failing. */
if (limited_targets) {
/* if all untested, success */
if (ret == CAIRO_TEST_UNTESTED) {
printf ("None of the tested backends passed, but tested targets are manually limited.\n"
"Passing the test, to not fail the suite.\n");
ret = CAIRO_TEST_SUCCESS;
}
/* if all passed, but expecting failure, return failure to not
* trigger an XPASS failure */
if (expectation == CAIRO_TEST_FAILURE && ret == CAIRO_TEST_SUCCESS) {
printf ("All tested backends passed, but tested targets are manually limited\n"
"and the test suite expects this test to fail for at least one target.\n"
"Intentionally failing the test, to not fail the suite.\n");
ret = CAIRO_TEST_FAILURE;
}
} else {
if (ret == CAIRO_TEST_UNTESTED)
ret = CAIRO_TEST_FAILURE;
}
cairo_test_fini ();
cairo_boilerplate_free_targets (targets_to_test);
return ret;
}
cairo_test_status_t
_cairo_test_context_run_for_target (cairo_test_context_t *ctx,
const cairo_boilerplate_target_t *target,
cairo_bool_t similar,
int dev_offset)
{
cairo_test_status_t status;
if (target->get_image_surface == NULL)
return CAIRO_TEST_UNTESTED;
if (similar && ! cairo_test_target_has_similar (ctx, target))
return CAIRO_TEST_UNTESTED;
cairo_test_log (ctx,
"Testing %s with %s%s target (dev offset %d)\n",
ctx->test_name,
similar ? " (similar) " : "",
target->name,
dev_offset);
printf ("%s.%s.%s [%d]%s:\t", ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content),
dev_offset,
similar ? " (similar)": "");
fflush (stdout);
#if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
if (! RUNNING_ON_VALGRIND) {
void (* volatile old_segfault_handler)(int);
void (* volatile old_segfpe_handler)(int);
void (* volatile old_sigpipe_handler)(int);
void (* volatile old_sigabrt_handler)(int);
void (* volatile old_sigalrm_handler)(int);
/* Set up a checkpoint to get back to in case of segfaults. */
#ifdef SIGSEGV
old_segfault_handler = signal (SIGSEGV, segfault_handler);
#endif
#ifdef SIGFPE
old_segfpe_handler = signal (SIGFPE, segfault_handler);
#endif
#ifdef SIGPIPE
old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
#endif
#ifdef SIGABRT
old_sigabrt_handler = signal (SIGABRT, segfault_handler);
#endif
#ifdef SIGALRM
old_sigalrm_handler = signal (SIGALRM, segfault_handler);
#endif
if (0 == setjmp (jmpbuf))
status = cairo_test_for_target (ctx, target, dev_offset, similar);
else
status = CAIRO_TEST_CRASHED;
#ifdef SIGSEGV
signal (SIGSEGV, old_segfault_handler);
#endif
#ifdef SIGFPE
signal (SIGFPE, old_segfpe_handler);
#endif
#ifdef SIGPIPE
signal (SIGPIPE, old_sigpipe_handler);
#endif
#ifdef SIGABRT
signal (SIGABRT, old_sigabrt_handler);
#endif
#ifdef SIGALRM
signal (SIGALRM, old_sigalrm_handler);
#endif
} else {
status = cairo_test_for_target (ctx, target, dev_offset, similar);
}
#else
status = cairo_test_for_target (ctx, target, dev_offset, similar);
#endif
cairo_test_log (ctx,
"TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content),
dev_offset, similar);
switch (status) {
case CAIRO_TEST_SUCCESS:
printf ("PASS\n");
cairo_test_log (ctx, "PASS\n");
break;
case CAIRO_TEST_UNTESTED:
printf ("UNTESTED\n");
cairo_test_log (ctx, "UNTESTED\n");
break;
default:
case CAIRO_TEST_CRASHED:
if (print_fail_on_stdout) {
printf ("!!!CRASHED!!!\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
cairo_test_log (ctx, "CRASHED\n");
fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
fail_face, normal_face);
break;
case CAIRO_TEST_ERROR:
if (print_fail_on_stdout) {
printf ("!!!ERROR!!!\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
cairo_test_log (ctx, "ERROR\n");
fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
fail_face, normal_face);
break;
case CAIRO_TEST_XFAILURE:
if (print_fail_on_stdout) {
printf ("XFAIL\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
xfail_face, normal_face);
cairo_test_log (ctx, "XFAIL\n");
break;
case CAIRO_TEST_NEW:
if (print_fail_on_stdout) {
printf ("NEW\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
fprintf (stderr, "%s.%s.%s [%d]%s:\t%sNEW%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
fail_face, normal_face);
cairo_test_log (ctx, "NEW\n");
break;
case CAIRO_TEST_NO_MEMORY:
case CAIRO_TEST_FAILURE:
if (print_fail_on_stdout) {
printf ("FAIL\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
fail_face, normal_face);
cairo_test_log (ctx, "FAIL\n");
break;
}
fflush (stdout);
return status;
}
