Esempio n. 1
0
static cairo_test_status_t
cairo_test_expecting (cairo_test_t *test,
		      cairo_test_status_t expectation)
{
    /* we use volatile here to make sure values are not clobbered
     * by longjmp */
    volatile size_t i, j, num_targets;
    volatile cairo_bool_t limited_targets = FALSE, print_fail_on_stdout = TRUE;
#ifdef HAVE_SIGNAL_H
    void (*old_segfault_handler)(int);
#endif
    volatile cairo_test_status_t status, ret;
    cairo_boilerplate_target_t ** volatile targets_to_test;

#ifdef HAVE_UNISTD_H
    if (isatty (2)) {
	fail_face = "\033[41m\033[37m\033[1m";
	normal_face = "\033[m";
	if (isatty (1))
	    print_fail_on_stdout = FALSE;
    }
#endif

    srcdir = getenv ("srcdir");
    if (!srcdir)
	srcdir = ".";

    cairo_test_init (test->name);
    printf ("%s\n", test->description);

    if (expectation == CAIRO_TEST_FAILURE)
    printf ("Expecting failure\n");

    {
	int tmp_num_targets;
	cairo_bool_t tmp_limited_targets;
	targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
	num_targets = tmp_num_targets;
	limited_targets = tmp_limited_targets;
    }

    /* The intended logic here is that we return overall SUCCESS
     * iff. there is at least one tested backend and that all tested
     * backends return SUCCESS, OR, there's backends were manually
     * limited, and none were tested.
     * In other words:
     *
     *  if      backends limited and no backend tested
     *          -> SUCCESS
     *	else if any backend not SUCCESS
     *		-> FAILURE
     *	else if all backends UNTESTED
     *		-> FAILURE
     *	else    (== some backend SUCCESS)
     *		-> SUCCESS
     *
     * Also, on a crash, run no further tests.
     */
    status = ret = CAIRO_TEST_UNTESTED;
    for (i = 0; i < num_targets && status != CAIRO_TEST_CRASHED; i++) {
	for (j = 0; j < NUM_DEVICE_OFFSETS; j++) {
	    cairo_boilerplate_target_t * volatile target = targets_to_test[i];
	    volatile int dev_offset = j * 25;

	    cairo_test_log ("Testing %s with %s target (dev offset %d)\n", test->name, target->name, dev_offset);
	    printf ("%s-%s-%s [%d]:\t", test->name, target->name,
		    cairo_boilerplate_content_name (target->content),
		    dev_offset);

#ifdef HAVE_SIGNAL_H
	    /* Set up a checkpoint to get back to in case of segfaults. */
	    old_segfault_handler = signal (SIGSEGV, segfault_handler);
	    if (0 == setjmp (jmpbuf))
#endif
		status = cairo_test_for_target (test, target, dev_offset);
#ifdef HAVE_SIGNAL_H
	    else
	        status = CAIRO_TEST_CRASHED;
	    signal (SIGSEGV, old_segfault_handler);
#endif

	    cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d RESULT: ",
			    test->name, target->name,
			    cairo_boilerplate_content_name (target->content),
			    dev_offset);

	    switch (status) {
	    case CAIRO_TEST_SUCCESS:
		printf ("PASS\n");
		cairo_test_log ("PASS\n");
		if (ret == CAIRO_TEST_UNTESTED)
		    ret = CAIRO_TEST_SUCCESS;
		break;
	    case CAIRO_TEST_UNTESTED:
		printf ("UNTESTED\n");
		cairo_test_log ("UNTESTED\n");
		break;
	    case CAIRO_TEST_CRASHED:
		if (print_fail_on_stdout) {
		    printf ("!!!CRASHED!!!\n");
		} else {
		    /* eat the test name */
		    printf ("\r");
		    fflush (stdout);
		}
		cairo_test_log ("CRASHED\n");
		fprintf (stderr, "%s-%s-%s [%d]:\t%s!!!CRASHED!!!%s\n",
			 test->name, target->name,
			 cairo_boilerplate_content_name (target->content), dev_offset,
			 fail_face, normal_face);
		ret = CAIRO_TEST_FAILURE;
		break;
	    default:
	    case CAIRO_TEST_FAILURE:
		if (expectation == CAIRO_TEST_FAILURE) {
		    printf ("XFAIL\n");
		    cairo_test_log ("XFAIL\n");
		} else {
		    if (print_fail_on_stdout) {
			printf ("FAIL\n");
		    } else {
			/* eat the test name */
			printf ("\r");
			fflush (stdout);
		    }
		    fprintf (stderr, "%s-%s-%s [%d]:\t%sFAIL%s\n",
			     test->name, target->name,
			     cairo_boilerplate_content_name (target->content), dev_offset,
			     fail_face, normal_face);
		    cairo_test_log ("FAIL\n");
		}
		ret = status;
		break;
	    }
	}
    }

    if (ret != CAIRO_TEST_SUCCESS)
        printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX);

    /* if the set of targets to test was limited using CAIRO_TEST_TARGET, we
     * behave slightly differently, to ensure that limiting the targets does
     * not increase the number of tests failing. */
    if (limited_targets) {

	/* if all untested, success */
	if (ret == CAIRO_TEST_UNTESTED) {
	    printf ("None of the tested backends passed, but tested targets are manually limited.\n"
		    "Passing the test, to not fail the suite.\n");
	    ret = CAIRO_TEST_SUCCESS;
	}

	/* if all passed, but expecting failure, return failure to not
	 * trigger an XPASS failure */
	if (expectation == CAIRO_TEST_FAILURE && ret == CAIRO_TEST_SUCCESS) {
	    printf ("All tested backends passed, but tested targets are manually limited\n"
		    "and the test suite expects this test to fail for at least one target.\n"
		    "Intentionally failing the test, to not fail the suite.\n");
	    ret = CAIRO_TEST_FAILURE;
	}

    } else {

	if (ret == CAIRO_TEST_UNTESTED)
	    ret = CAIRO_TEST_FAILURE;

    }

    cairo_test_fini ();

    cairo_boilerplate_free_targets (targets_to_test);

    return ret;
}
Esempio n. 2
0
static cairo_test_status_t
cairo_test_for_target (cairo_test_t			 *test,
		       cairo_boilerplate_target_t	 *target,
		       int				  dev_offset)
{
    cairo_test_status_t status;
    cairo_surface_t *surface = NULL;
    cairo_t *cr;
    char *png_name, *ref_name, *diff_name, *offset_str;
    cairo_test_status_t ret = CAIRO_TEST_SUCCESS;
    cairo_content_t expected_content;
    cairo_font_options_t *font_options;
    const char *format;

    /* Get the strings ready that we'll need. */
    format = cairo_boilerplate_content_name (target->content);
    if (dev_offset)
	xasprintf (&offset_str, "-%d", dev_offset);
    else
	offset_str = strdup("");

    xasprintf (&png_name, "%s-%s-%s%s%s",
	       test->name,
	       target->name,
	       format,
	       offset_str, CAIRO_TEST_PNG_SUFFIX);
    ref_name = cairo_ref_name_for_test_target_format (test->name, target->name, format);
    xasprintf (&diff_name, "%s-%s-%s%s%s",
	       test->name,
	       target->name,
	       format,
	       offset_str, CAIRO_TEST_DIFF_SUFFIX);

    if (target->is_vector) {
	int i;

	for (i = 0; vector_ignored_tests[i] != NULL; i++)
	    if (strcmp (test->name, vector_ignored_tests[i]) == 0) {
		cairo_test_log ("Error: Skipping for vector target %s\n", target->name);
		ret = CAIRO_TEST_UNTESTED;
		goto UNWIND_STRINGS;
	    }
    }

    if (ret == CAIRO_TEST_SUCCESS) {
	/* Run the actual drawing code. */

	if (test->width && test->height) {
	    test->width += dev_offset;
	    test->height += dev_offset;
	}

	surface = (target->create_surface) (test->name,
					    target->content,
					    test->width,
					    test->height,
					    CAIRO_BOILERPLATE_MODE_TEST,
					    &target->closure);

	if (test->width && test->height) {
	    test->width -= dev_offset;
	    test->height -= dev_offset;;
	}
    }

    if (surface == NULL) {
	cairo_test_log ("Error: Failed to set %s target\n", target->name);
	ret = CAIRO_TEST_UNTESTED;
	goto UNWIND_STRINGS;
    }

    /* Check that we created a surface of the expected type. */
    if (cairo_surface_get_type (surface) != target->expected_type) {
	cairo_test_log ("Error: Created surface is of type %d (expected %d)\n",
			cairo_surface_get_type (surface), target->expected_type);
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_SURFACE;
    }

    /* Check that we created a surface of the expected content,
     * (ignore the articifical
     * CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
     */
    expected_content = target->content;
    if (expected_content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED)
	expected_content = CAIRO_CONTENT_COLOR_ALPHA;

    if (cairo_surface_get_content (surface) != expected_content) {
	cairo_test_log ("Error: Created surface has content %d (expected %d)\n",
			cairo_surface_get_content (surface), expected_content);
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_SURFACE;
    }

    cairo_surface_set_device_offset (surface, dev_offset, dev_offset);

    cr = cairo_create (surface);

    /* Clear to transparent (or black) depending on whether the target
     * surface supports alpha. */
    cairo_save (cr);
    cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
    cairo_paint (cr);
    cairo_restore (cr);

    /* Set all components of font_options to avoid backend differences
     * and reduce number of needed reference images. */
    font_options = cairo_font_options_create ();
    cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
    cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
    cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
    cairo_set_font_options (cr, font_options);
    cairo_font_options_destroy (font_options);

    status = (test->draw) (cr, test->width, test->height);

    /* Then, check all the different ways it could fail. */
    if (status) {
	cairo_test_log ("Error: Function under test failed\n");
	ret = status;
	goto UNWIND_CAIRO;
    }

    if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
	cairo_test_log ("Error: Function under test left cairo status in an error state: %s\n",
			cairo_status_to_string (cairo_status (cr)));
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_CAIRO;
    }

    /* Skip image check for tests with no image (width,height == 0,0) */
    if (test->width != 0 && test->height != 0) {
	buffer_diff_result_t result;
	cairo_status_t diff_status;
	xunlink (png_name);
	(target->write_to_png) (surface, png_name);

	if (!ref_name) {
	    cairo_test_log ("Error: Cannot find reference image for %s/%s-%s-%s%s\n",srcdir,
			    test->name,
			    target->name,
			    format,
			    CAIRO_TEST_REF_SUFFIX);
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}

	cairo_test_log ("Comparing result against reference image: %s\n", ref_name);

	if (target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED) {
	    diff_status= image_diff_flattened (png_name, ref_name, diff_name,
					       dev_offset, dev_offset, 0, 0, &result);
	} else {
	    diff_status = image_diff (png_name, ref_name, diff_name,
				      dev_offset, dev_offset, 0, 0, &result);
	}
	if (diff_status) {
	    cairo_test_log ("Error: Failed to compare images: %s\n",
			    cairo_status_to_string (diff_status));
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}
	if (result.pixels_changed && result.max_diff > target->error_tolerance) {
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}
    }

UNWIND_CAIRO:
    cairo_destroy (cr);
UNWIND_SURFACE:
    cairo_surface_destroy (surface);

    cairo_debug_reset_static_data ();

    if (target->cleanup)
	target->cleanup (target->closure);

UNWIND_STRINGS:
    if (png_name)
      free (png_name);
    if (ref_name)
      free (ref_name);
    if (diff_name)
      free (diff_name);
    if (offset_str)
      free (offset_str);

    return ret;
}
static cairo_test_status_t
preamble (cairo_test_context_t *ctx)
{
    cairo_t *cr;
    cairo_test_status_t ret = CAIRO_TEST_UNTESTED;
    struct {
	double x, y;
    } ppi[] = {
	{ 600, 600 },
	{ 600, 72 },

	{ 300, 300 },
	{ 300, 72 },

	{ 150, 150 },
	{ 150, 72 },

	{ 75, 75 },
	{ 75, 72 },

	{ 72, 600 },
	{ 72, 300 },
	{ 72, 150 },
	{ 72, 75 },
	{ 72, 72 },
	{ 72, 37.5 },

	{ 37.5, 72 },
	{ 37.5, 37.5 },
    };
    unsigned int i;
    int n, num_ppi;

    num_ppi = sizeof (ppi) / sizeof (ppi[0]);

#if GENERATE_REFERENCE
    for (n = 0; n < num_ppi; n++) {
	char *ref_name;
	xasprintf (&ref_name, "fallback-resolution.ppi%gx%g.ref.png",
		   ppi[n].x, ppi[n].y);
	generate_reference (ppi[n].x, ppi[n].y, ref_name);
	free (ref_name);
    }
#endif

    for (i = 0; i < ctx->num_targets; i++) {
	const cairo_boilerplate_target_t *target = ctx->targets_to_test[i];
	cairo_surface_t *surface = NULL;
	char *base_name;
	void *closure;
	const char *format;
	cairo_status_t status;

	if (! target->is_vector)
	    continue;

	if (! cairo_test_is_target_enabled (ctx, target->name))
	    continue;

	format = cairo_boilerplate_content_name (target->content);
	xasprintf (&base_name, "fallback-resolution.%s.%s",
		   target->name,
		   format);

	surface = (target->create_surface) (base_name,
					    target->content,
					    SIZE, SIZE,
					    SIZE, SIZE,
					    CAIRO_BOILERPLATE_MODE_TEST,
					    0,
					    &closure);

	if (surface == NULL) {
	    free (base_name);
	    continue;
	}

	if (ret == CAIRO_TEST_UNTESTED)
	    ret = CAIRO_TEST_SUCCESS;

	cairo_surface_destroy (surface);
	if (target->cleanup)
	    target->cleanup (closure);
	free (base_name);

	/* we need to recreate the surface for each resolution as we include
	 * SVG in testing which does not support the paginated interface.
	 */
	for (n = 0; n < num_ppi; n++) {
	    char *test_name;
	    cairo_bool_t pass;

	    xasprintf (&test_name, "fallback-resolution.ppi%gx%g",
		       ppi[n].x, ppi[n].y);
	    xasprintf (&base_name, "%s.%s.%s",
		       test_name,
		       target->name,
		       format);

	    surface = (target->create_surface) (base_name,
						target->content,
						SIZE + 25, SIZE + 25,
						SIZE + 25, SIZE + 25,
						CAIRO_BOILERPLATE_MODE_TEST,
						0,
						&closure);
	    if (surface == NULL || cairo_surface_status (surface)) {
		cairo_test_log (ctx, "Failed to generate surface: %s.%s\n",
				target->name,
				format);
		free (base_name);
		free (test_name);
		ret = CAIRO_TEST_FAILURE;
		continue;
	    }

	    cairo_test_log (ctx,
			    "Testing fallback-resolution %gx%g with %s target\n",
			    ppi[n].x, ppi[n].y, target->name);
	    printf ("%s:\t", base_name);
	    fflush (stdout);

	    if (target->force_fallbacks != NULL)
		target->force_fallbacks (surface, ~0U);
	    cr = cairo_create (surface);
#if SET_TOLERANCE
	    cairo_set_tolerance (cr, 3.0);
#endif

	    cairo_surface_set_device_offset (surface, 25, 25);
	    cairo_surface_set_fallback_resolution (surface,
						   ppi[n].x, ppi[n].y);

	    cairo_save (cr); {
		cairo_set_source_rgb (cr, 1, 1, 1);
		cairo_paint (cr);
	    } cairo_restore (cr);

	    /* First draw the top half in a conventional way. */
	    cairo_save (cr); {
		cairo_rectangle (cr, 0, 0, SIZE, SIZE / 2.0);
		cairo_clip (cr);

		draw (cr, SIZE, SIZE);
	    } cairo_restore (cr);

	    /* Then draw the bottom half in a separate group,
	     * (exposing a bug in 1.6.4 with the group not being
	     * rendered with the correct fallback resolution). */
	    cairo_save (cr); {
		cairo_rectangle (cr, 0, SIZE / 2.0, SIZE, SIZE / 2.0);
		cairo_clip (cr);

		cairo_push_group (cr); {
		    draw (cr, SIZE, SIZE);
		} cairo_pop_group_to_source (cr);

		cairo_paint (cr);
	    } cairo_restore (cr);

	    status = cairo_status (cr);
	    cairo_destroy (cr);

	    pass = FALSE;
	    if (status) {
		cairo_test_log (ctx, "Error: Failed to create target surface: %s\n",
				cairo_status_to_string (status));
		ret = CAIRO_TEST_FAILURE;
	    } else {
		/* extract the image and compare it to our reference */
		if (! check_result (ctx, target, test_name, base_name, surface))
		    ret = CAIRO_TEST_FAILURE;
		else
		    pass = TRUE;
	    }
	    cairo_surface_destroy (surface);
	    if (target->cleanup)
		target->cleanup (closure);

	    free (base_name);
	    free (test_name);

	    if (pass) {
		printf ("PASS\n");
	    } else {
		printf ("FAIL\n");
	    }
	    fflush (stdout);
	}
    }

    return ret;
}
static cairo_bool_t
check_result (cairo_test_context_t *ctx,
	      const cairo_boilerplate_target_t *target,
	      const char *test_name,
	      const char *base_name,
	      cairo_surface_t *surface)
{
    const char *format;
    char *ref_name;
    char *png_name;
    char *diff_name;
    cairo_surface_t *test_image, *ref_image, *diff_image;
    buffer_diff_result_t result;
    cairo_status_t status;
    cairo_bool_t ret;

    /* XXX log target, OUTPUT, REFERENCE, DIFFERENCE for index.html */

    if (target->finish_surface != NULL) {
	status = target->finish_surface (surface);
	if (status) {
	    cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
		    cairo_status_to_string (status));
	    cairo_surface_destroy (surface);
	    return FALSE;
	}
    }

    xasprintf (&png_name,  "%s.out.png", base_name);
    xasprintf (&diff_name, "%s.diff.png", base_name);

    test_image = target->get_image_surface (surface, 0, SIZE, SIZE);
    if (cairo_surface_status (test_image)) {
	cairo_test_log (ctx, "Error: Failed to extract page: %s\n",
		        cairo_status_to_string (cairo_surface_status (test_image)));
	cairo_surface_destroy (test_image);
	free (png_name);
	free (diff_name);
	return FALSE;
    }

    _xunlink (ctx, png_name);
    status = cairo_surface_write_to_png (test_image, png_name);
    if (status) {
	cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
		cairo_status_to_string (status));
	cairo_surface_destroy (test_image);
	free (png_name);
	free (diff_name);
	return FALSE;
    }

    format = cairo_boilerplate_content_name (target->content);
    ref_name = cairo_test_reference_filename (ctx,
					      base_name,
					      test_name,
					      target->name,
					      target->basename,
					      format,
					      CAIRO_TEST_REF_SUFFIX,
					      CAIRO_TEST_PNG_EXTENSION);
    if (ref_name == NULL) {
	cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
		        base_name);
	cairo_surface_destroy (test_image);
	free (png_name);
	free (diff_name);
	return FALSE;
    }


    ref_image = cairo_test_get_reference_image (ctx, ref_name,
	    target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
    if (cairo_surface_status (ref_image)) {
	cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
		        ref_name,
		cairo_status_to_string (cairo_surface_status (ref_image)));
	cairo_surface_destroy (ref_image);
	cairo_surface_destroy (test_image);
	free (png_name);
	free (diff_name);
	free (ref_name);
	return FALSE;
    }

    diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
	    SIZE, SIZE);

    ret = TRUE;
    status = image_diff (ctx,
	    test_image, ref_image, diff_image,
	    &result);
    _xunlink (ctx, diff_name);
    if (status) {
	cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
			cairo_status_to_string (status));
	ret = FALSE;
    } else if (result.pixels_changed &&
	       result.max_diff > target->error_tolerance)
    {
	ret = FALSE;

	status = cairo_surface_write_to_png (diff_image, diff_name);
	if (status) {
	    cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
		    cairo_status_to_string (status));
	}
    }

    cairo_surface_destroy (test_image);
    cairo_surface_destroy (diff_image);
    free (png_name);
    free (diff_name);
    free (ref_name);

    return ret;
}
Esempio n. 5
0
static cairo_test_status_t
cairo_test_for_target (cairo_test_context_t		 *ctx,
		       const cairo_boilerplate_target_t	 *target,
		       int				  dev_offset,
		       cairo_bool_t                       similar)
{
    cairo_test_status_t status;
    cairo_surface_t *surface = NULL;
    cairo_t *cr;
    const char *empty_str = "";
    char *offset_str;
    char *base_name, *base_path;
    char *out_png_path;
    char *ref_path = NULL, *ref_png_path, *cmp_png_path = NULL;
    char *new_path = NULL, *new_png_path;
    char *xfail_path = NULL, *xfail_png_path;
    char *base_ref_png_path;
    char *base_new_png_path;
    char *base_xfail_png_path;
    char *diff_png_path;
    char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
    cairo_test_status_t ret;
    cairo_content_t expected_content;
    cairo_font_options_t *font_options;
    const char *format;
    cairo_bool_t have_output = FALSE;
    cairo_bool_t have_result = FALSE;
    void *closure;
    double width, height;
    cairo_bool_t have_output_dir;
#if HAVE_MEMFAULT
    int malloc_failure_iterations = ctx->malloc_failure;
    int last_fault_count = 0;
#endif

    /* Get the strings ready that we'll need. */
    format = cairo_boilerplate_content_name (target->content);
    if (dev_offset)
	xasprintf (&offset_str, ".%d", dev_offset);
    else
	offset_str = (char *) empty_str;

    xasprintf (&base_name, "%s.%s.%s%s%s",
	       ctx->test_name,
	       target->name,
	       format,
	       similar ? ".similar" : "",
	       offset_str);

    if (offset_str != empty_str)
      free (offset_str);

    ref_png_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  target->name,
						  target->basename,
						  format,
						  CAIRO_TEST_REF_SUFFIX,
						  CAIRO_TEST_PNG_EXTENSION);
    new_png_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  target->name,
						  target->basename,
						  format,
						  CAIRO_TEST_NEW_SUFFIX,
						  CAIRO_TEST_PNG_EXTENSION);
    xfail_png_path = cairo_test_reference_filename (ctx,
						    base_name,
						    ctx->test_name,
						    target->name,
						    target->basename,
						    format,
						    CAIRO_TEST_XFAIL_SUFFIX,
						    CAIRO_TEST_PNG_EXTENSION);

    base_ref_png_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  NULL, NULL,
						  format,
						  CAIRO_TEST_REF_SUFFIX,
						  CAIRO_TEST_PNG_EXTENSION);
    base_new_png_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  NULL, NULL,
						  format,
						  CAIRO_TEST_NEW_SUFFIX,
						  CAIRO_TEST_PNG_EXTENSION);
    base_xfail_png_path = cairo_test_reference_filename (ctx,
						    base_name,
						    ctx->test_name,
						    NULL, NULL,
						    format,
						    CAIRO_TEST_XFAIL_SUFFIX,
						    CAIRO_TEST_PNG_EXTENSION);

    if (target->file_extension != NULL) {
	ref_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  target->name,
						  target->basename,
						  format,
						  CAIRO_TEST_REF_SUFFIX,
						  target->file_extension);
	new_path = cairo_test_reference_filename (ctx,
						  base_name,
						  ctx->test_name,
						  target->name,
						  target->basename,
						  format,
						  CAIRO_TEST_NEW_SUFFIX,
						  target->file_extension);
	xfail_path = cairo_test_reference_filename (ctx,
						    base_name,
						    ctx->test_name,
						    target->name,
						    target->basename,
						    format,
						    CAIRO_TEST_XFAIL_SUFFIX,
						    target->file_extension);
    }

    have_output_dir = _cairo_test_mkdir (ctx->output);
    xasprintf (&base_path, "%s/%s",
	       have_output_dir ? ctx->output : ".",
	       base_name);
    xasprintf (&out_png_path, "%s" CAIRO_TEST_OUT_PNG, base_path);
    xasprintf (&diff_png_path, "%s" CAIRO_TEST_DIFF_PNG, base_path);

    if (ctx->test->requirements != NULL) {
	const char *required;

	required = target->is_vector ? "target=raster" : "target=vector";
	if (strstr (ctx->test->requirements, required) != NULL) {
	    cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
			    target->is_vector ? "vector" : "raster",
			    target->name);
	    ret = CAIRO_TEST_UNTESTED;
	    goto UNWIND_STRINGS;
	}

	required = target->is_recording ? "target=!recording" : "target=recording";
	if (strstr (ctx->test->requirements, required) != NULL) {
	    cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
			    target->is_recording ? "recording" : "non-recording",
			    target->name);
	    ret = CAIRO_TEST_UNTESTED;
	    goto UNWIND_STRINGS;
	}
    }

    width = ctx->test->width;
    height = ctx->test->height;
    if (width && height) {
	width += dev_offset;
	height += dev_offset;
    }

#if HAVE_MEMFAULT
REPEAT:
    MEMFAULT_CLEAR_FAULTS ();
    MEMFAULT_RESET_LEAKS ();
    ctx->last_fault_count = 0;
    last_fault_count = MEMFAULT_COUNT_FAULTS ();

    /* Pre-initialise fontconfig so that the configuration is loaded without
     * malloc failures (our primary goal is to test cairo fault tolerance).
     */
#if HAVE_FCINIT
    FcInit ();
#endif

    MEMFAULT_ENABLE_FAULTS ();
#endif
    have_output = FALSE;
    have_result = FALSE;

    /* Run the actual drawing code. */
    ret = CAIRO_TEST_SUCCESS;
    surface = (target->create_surface) (base_path,
					target->content,
					width, height,
					ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
					ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
					CAIRO_BOILERPLATE_MODE_TEST,
					&closure);
    if (surface == NULL) {
	cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
	ret = CAIRO_TEST_UNTESTED;
	goto UNWIND_STRINGS;
    }

#if HAVE_MEMFAULT
    if (ctx->malloc_failure &&
	MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
	cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY)
    {
	goto REPEAT;
    }
#endif

    if (cairo_surface_status (surface)) {
	MF (MEMFAULT_PRINT_FAULTS ());
	cairo_test_log (ctx, "Error: Created an error surface: %s\n",
			cairo_status_to_string (cairo_surface_status (surface)));
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_STRINGS;
    }

    /* Check that we created a surface of the expected type. */
    if (cairo_surface_get_type (surface) != target->expected_type) {
	MF (MEMFAULT_PRINT_FAULTS ());
	cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
			cairo_surface_get_type (surface), target->expected_type);
	ret = CAIRO_TEST_UNTESTED;
	goto UNWIND_SURFACE;
    }

    /* Check that we created a surface of the expected content,
     * (ignore the artificial CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
     */
    expected_content = cairo_boilerplate_content (target->content);

    if (cairo_surface_get_content (surface) != expected_content) {
	MF (MEMFAULT_PRINT_FAULTS ());
	cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
			cairo_surface_get_content (surface), expected_content);
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_SURFACE;
    }

    if (cairo_surface_set_user_data (surface,
				     &cairo_boilerplate_output_basename_key,
				     base_path,
				     NULL))
    {
#if HAVE_MEMFAULT
	cairo_surface_destroy (surface);

	if (target->cleanup)
	    target->cleanup (closure);

	goto REPEAT;
#else
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_SURFACE;
#endif
    }

    cairo_surface_set_device_offset (surface, dev_offset, dev_offset);

    cr = cairo_create (surface);
    if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
#if HAVE_MEMFAULT
	cairo_destroy (cr);
	cairo_surface_destroy (surface);

	if (target->cleanup)
	    target->cleanup (closure);

	goto REPEAT;
#else
	ret = CAIRO_TEST_FAILURE;
	goto UNWIND_CAIRO;
#endif
    }

    if (similar)
	cairo_push_group_with_content (cr, expected_content);

    /* Clear to transparent (or black) depending on whether the target
     * surface supports alpha. */
    cairo_save (cr);
    cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
    cairo_paint (cr);
    cairo_restore (cr);

    /* Set all components of font_options to avoid backend differences
     * and reduce number of needed reference images. */
    font_options = cairo_font_options_create ();
    cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
    cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
    cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
    cairo_set_font_options (cr, font_options);
    cairo_font_options_destroy (font_options);

    cairo_save (cr);
    alarm (ctx->timeout);
    status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
    alarm (0);
    cairo_restore (cr);

    if (similar) {
	cairo_pop_group_to_source (cr);
	cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
	cairo_paint (cr);
    }

#if HAVE_MEMFAULT
    MEMFAULT_DISABLE_FAULTS ();

    /* repeat test after malloc failure injection */
    if (ctx->malloc_failure &&
	MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
	(status == CAIRO_TEST_NO_MEMORY ||
	 cairo_status (cr) == CAIRO_STATUS_NO_MEMORY ||
	 cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY))
    {
	cairo_destroy (cr);
	cairo_surface_destroy (surface);
	if (target->cleanup)
	    target->cleanup (closure);
	cairo_debug_reset_static_data ();
#if HAVE_FCFINI
	FcFini ();
#endif
	if (MEMFAULT_COUNT_LEAKS () > 0) {
	    MEMFAULT_PRINT_FAULTS ();
	    MEMFAULT_PRINT_LEAKS ();
	}

	goto REPEAT;
    }
#endif

    /* Then, check all the different ways it could fail. */
    if (status) {
	cairo_test_log (ctx, "Error: Function under test failed\n");
	ret = status;
	goto UNWIND_CAIRO;
    }

#if HAVE_MEMFAULT
    if (MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
	MEMFAULT_HAS_FAULTS ())
    {
	VALGRIND_PRINTF ("Unreported memfaults...");
	MEMFAULT_PRINT_FAULTS ();
    }
#endif

    if (target->finish_surface != NULL) {
#if HAVE_MEMFAULT
	/* We need to re-enable faults as most recording-surface processing
	 * is done during cairo_surface_finish().
	 */
	MEMFAULT_CLEAR_FAULTS ();
	last_fault_count = MEMFAULT_COUNT_FAULTS ();
	MEMFAULT_ENABLE_FAULTS ();
#endif

	/* also check for infinite loops whilst replaying */
	alarm (ctx->timeout);
	status = target->finish_surface (surface);
	alarm (0);

#if HAVE_MEMFAULT
	MEMFAULT_DISABLE_FAULTS ();

	if (ctx->malloc_failure &&
	    MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
	    status == CAIRO_STATUS_NO_MEMORY)
	{
	    cairo_destroy (cr);
	    cairo_surface_destroy (surface);
	    if (target->cleanup)
		target->cleanup (closure);
	    cairo_debug_reset_static_data ();
#if HAVE_FCFINI
	    FcFini ();
#endif
	    if (MEMFAULT_COUNT_LEAKS () > 0) {
		MEMFAULT_PRINT_FAULTS ();
		MEMFAULT_PRINT_LEAKS ();
	    }

	    goto REPEAT;
	}
#endif
	if (status) {
	    cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
			    cairo_status_to_string (status));
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}
    }

    /* Skip image check for tests with no image (width,height == 0,0) */
    if (ctx->test->width != 0 && ctx->test->height != 0) {
	cairo_surface_t *ref_image;
	cairo_surface_t *test_image;
	cairo_surface_t *diff_image;
	buffer_diff_result_t result;
	cairo_status_t diff_status;

	if (ref_png_path == NULL) {
	    cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
			    base_name);

	    /* we may be running this test to generate reference images */
	    _xunlink (ctx, out_png_path);
	    /* be more generous as we may need to use external renderers */
	    alarm (4 * ctx->timeout);
	    test_image = target->get_image_surface (surface, 0,
		                                    ctx->test->width,
						    ctx->test->height);
	    alarm (0);
	    diff_status = cairo_surface_write_to_png (test_image, out_png_path);
	    cairo_surface_destroy (test_image);
	    if (diff_status) {
		if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
		    ret = CAIRO_TEST_CRASHED;
		else
		    ret = CAIRO_TEST_FAILURE;
		cairo_test_log (ctx,
			        "Error: Failed to write output image: %s\n",
			        cairo_status_to_string (diff_status));
	    }
	    have_output = TRUE;

	    ret = CAIRO_TEST_XFAILURE;
	    goto UNWIND_CAIRO;
	}

	if (target->file_extension != NULL) { /* compare vector surfaces */
	    char *filenames[] = {
		ref_png_path,
		ref_path,
		new_png_path,
		new_path,
		xfail_png_path,
		xfail_path,
		base_ref_png_path,
		base_new_png_path,
		base_xfail_png_path,
	    };

	    xasprintf (&test_filename, "%s.out%s",
		       base_path, target->file_extension);
	    xasprintf (&pass_filename, "%s.pass%s",
		       base_path, target->file_extension);
	    xasprintf (&fail_filename, "%s.fail%s",
		       base_path, target->file_extension);

	    if (cairo_test_file_is_older (pass_filename,
					  filenames,
					  ARRAY_SIZE (filenames)))
	    {
		_xunlink (ctx, pass_filename);
	    }
	    if (cairo_test_file_is_older (fail_filename,
					  filenames,
					  ARRAY_SIZE (filenames)))
	    {
		_xunlink (ctx, fail_filename);
	    }

	    if (cairo_test_files_equal (out_png_path, ref_path)) {
		cairo_test_log (ctx, "Vector surface matches reference.\n");
		have_output = FALSE;
		ret = CAIRO_TEST_SUCCESS;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, new_path)) {
		cairo_test_log (ctx, "Vector surface matches current failure.\n");
		have_output = FALSE;
		ret = CAIRO_TEST_NEW;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, xfail_path)) {
		cairo_test_log (ctx, "Vector surface matches known failure.\n");
		have_output = FALSE;
		ret = CAIRO_TEST_XFAILURE;
		goto UNWIND_CAIRO;
	    }

	    if (cairo_test_files_equal (test_filename, pass_filename)) {
		/* identical output as last known PASS */
		cairo_test_log (ctx, "Vector surface matches last pass.\n");
		have_output = TRUE;
		ret = CAIRO_TEST_SUCCESS;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (test_filename, fail_filename)) {
		/* identical output as last known FAIL, fail */
		cairo_test_log (ctx, "Vector surface matches last fail.\n");
		have_result = TRUE; /* presume these were kept around as well */
		have_output = TRUE;
		ret = CAIRO_TEST_FAILURE;
		goto UNWIND_CAIRO;
	    }
	}

	/* be more generous as we may need to use external renderers */
	alarm (4 * ctx->timeout);
	test_image = target->get_image_surface (surface, 0,
					        ctx->test->width,
						ctx->test->height);
	alarm (0);
	if (cairo_surface_status (test_image)) {
	    cairo_test_log (ctx, "Error: Failed to extract image: %s\n",
			    cairo_status_to_string (cairo_surface_status (test_image)));
	    if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
		ret = CAIRO_TEST_CRASHED;
	    else
		ret = CAIRO_TEST_FAILURE;
	    cairo_surface_destroy (test_image);
	    goto UNWIND_CAIRO;
	}

	_xunlink (ctx, out_png_path);
	diff_status = cairo_surface_write_to_png (test_image, out_png_path);
	if (diff_status) {
	    cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
			    cairo_status_to_string (diff_status));
	    cairo_surface_destroy (test_image);
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}
	have_output = TRUE;

	/* binary compare png files (no decompression) */
	if (target->file_extension == NULL) {
	    char *filenames[] = {
		ref_png_path,
		new_png_path,
		xfail_png_path,
		base_ref_png_path,
		base_new_png_path,
		base_xfail_png_path,
	    };

	    xasprintf (&test_filename, "%s", out_png_path);
	    xasprintf (&pass_filename, "%s.pass.png", base_path);
	    xasprintf (&fail_filename, "%s.fail.png", base_path);

	    if (cairo_test_file_is_older (pass_filename,
					  filenames,
					  ARRAY_SIZE (filenames)))
	    {
		_xunlink (ctx, pass_filename);
	    }
	    if (cairo_test_file_is_older (fail_filename,
					  filenames,
					  ARRAY_SIZE (filenames)))
	    {
		_xunlink (ctx, fail_filename);
	    }

	    if (cairo_test_files_equal (test_filename, pass_filename)) {
		cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
                have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_SUCCESS;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, ref_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
                have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_SUCCESS;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, new_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
                have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_NEW;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
                have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_XFAILURE;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (test_filename, fail_filename)) {
		cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
		have_result = TRUE; /* presume these were kept around as well */
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_FAILURE;
		goto UNWIND_CAIRO;
	    }
	} else {
	    if (cairo_test_files_equal (out_png_path, ref_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
		have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_SUCCESS;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, new_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
		have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_NEW;
		goto UNWIND_CAIRO;
	    }
	    if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
		cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
		have_result = TRUE;
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_XFAILURE;
		goto UNWIND_CAIRO;
	    }
	}

	if (cairo_test_files_equal (out_png_path, base_ref_png_path)) {
	    cairo_test_log (ctx, "PNG file exactly reference image.\n");
	    have_result = TRUE;
	    cairo_surface_destroy (test_image);
	    ret = CAIRO_TEST_SUCCESS;
	    goto UNWIND_CAIRO;
	}
	if (cairo_test_files_equal (out_png_path, base_new_png_path)) {
	    cairo_test_log (ctx, "PNG file exactly current failure image.\n");
	    have_result = TRUE;
	    cairo_surface_destroy (test_image);
	    ret = CAIRO_TEST_NEW;
	    goto UNWIND_CAIRO;
	}
	if (cairo_test_files_equal (out_png_path, base_xfail_png_path)) {
	    cairo_test_log (ctx, "PNG file exactly known failure image.\n");
	    have_result = TRUE;
	    cairo_surface_destroy (test_image);
	    ret = CAIRO_TEST_XFAILURE;
	    goto UNWIND_CAIRO;
	}

	/* first compare against the ideal reference */
	ref_image = cairo_test_get_reference_image (ctx, base_ref_png_path,
						    target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
	if (cairo_surface_status (ref_image)) {
	    cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
			    base_ref_png_path,
			    cairo_status_to_string (cairo_surface_status (ref_image)));
	    cairo_surface_destroy (test_image);
	    ret = CAIRO_TEST_FAILURE;
	    goto UNWIND_CAIRO;
	}

	diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
						 ctx->test->width,
						 ctx->test->height);

	cmp_png_path = base_ref_png_path;
	diff_status = image_diff (ctx,
				  test_image, ref_image, diff_image,
				  &result);
	_xunlink (ctx, diff_png_path);
	if (diff_status ||
            image_diff_is_failure (&result, target->error_tolerance))
	{
	    /* that failed, so check against the specific backend */
	    ref_image = cairo_test_get_reference_image (ctx, ref_png_path,
							target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
	    if (cairo_surface_status (ref_image)) {
		cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
				ref_png_path,
				cairo_status_to_string (cairo_surface_status (ref_image)));
		cairo_surface_destroy (test_image);
		ret = CAIRO_TEST_FAILURE;
		goto UNWIND_CAIRO;
	    }

	    cmp_png_path = ref_png_path;
	    diff_status = image_diff (ctx,
				      test_image, ref_image,
				      diff_image,
				      &result);
	    if (diff_status)
	    {
		cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
				cairo_status_to_string (diff_status));
		ret = CAIRO_TEST_FAILURE;
	    }
	    else if (image_diff_is_failure (&result, target->error_tolerance))
	    {
		ret = CAIRO_TEST_FAILURE;

		diff_status = cairo_surface_write_to_png (diff_image,
							  diff_png_path);
		if (diff_status) {
		    cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
				    cairo_status_to_string (diff_status));
		} else {
		    have_result = TRUE;
		}

		cairo_test_copy_file (test_filename, fail_filename);
	    }
	    else
	    { /* success */
		cairo_test_copy_file (test_filename, pass_filename);
	    }
	}
	else
	{ /* success */
	    cairo_test_copy_file (test_filename, pass_filename);
	}

	/* If failed, compare against the current image output,
	 * and attempt to detect systematic failures.
	 */
	if (ret == CAIRO_TEST_FAILURE) {
	    char *image_out_path;

	    image_out_path =
		cairo_test_reference_filename (ctx,
					       base_name,
					       ctx->test_name,
					       "image",
					       "image",
					       format,
					       CAIRO_TEST_OUT_SUFFIX,
					       CAIRO_TEST_PNG_EXTENSION);
	    if (image_out_path != NULL) {
		if (cairo_test_files_equal (out_png_path,
					    image_out_path))
		{
		    ret = CAIRO_TEST_XFAILURE;
		}
		else
		{
		    ref_image =
			cairo_image_surface_create_from_png (image_out_path);
		    if (cairo_surface_status (ref_image) == CAIRO_STATUS_SUCCESS)
		    {
			diff_status = image_diff (ctx,
						  test_image, ref_image,
						  diff_image,
						  &result);
			if (diff_status == CAIRO_STATUS_SUCCESS &&
			    !image_diff_is_failure (&result, target->error_tolerance))
			{
			    ret = CAIRO_TEST_XFAILURE;
			}

			cairo_surface_destroy (ref_image);
		    }
		}

		free (image_out_path);
	    }
	}

	cairo_surface_destroy (test_image);
	cairo_surface_destroy (diff_image);
    }

    if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
	cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
			cairo_status_to_string (cairo_status (cr)));
	ret = CAIRO_TEST_ERROR;
	goto UNWIND_CAIRO;
    }

UNWIND_CAIRO:
    free (test_filename);
    free (fail_filename);
    free (pass_filename);

    test_filename = fail_filename = pass_filename = NULL;

#if HAVE_MEMFAULT
    if (ret == CAIRO_TEST_FAILURE)
	MEMFAULT_PRINT_FAULTS ();
#endif
    cairo_destroy (cr);
UNWIND_SURFACE:
    cairo_surface_destroy (surface);

    if (target->cleanup)
	target->cleanup (closure);

#if HAVE_MEMFAULT
    cairo_debug_reset_static_data ();

#if HAVE_FCFINI
    FcFini ();
#endif

    if (MEMFAULT_COUNT_LEAKS () > 0) {
	if (ret != CAIRO_TEST_FAILURE)
	    MEMFAULT_PRINT_FAULTS ();
	MEMFAULT_PRINT_LEAKS ();
    }

    if (ret == CAIRO_TEST_SUCCESS && --malloc_failure_iterations > 0)
	goto REPEAT;
#endif

    if (have_output)
	cairo_test_log (ctx, "OUTPUT: %s\n", out_png_path);

    if (have_result) {
	if (cmp_png_path == NULL) {
	    /* XXX presume we matched the normal ref last time */
	    cmp_png_path = ref_png_path;
	}
	cairo_test_log (ctx,
			"REFERENCE: %s\nDIFFERENCE: %s\n",
			cmp_png_path, diff_png_path);
    }

UNWIND_STRINGS:
    free (out_png_path);
    free (ref_png_path);
    free (base_ref_png_path);
    free (ref_path);
    free (new_png_path);
    free (base_new_png_path);
    free (new_path);
    free (xfail_png_path);
    free (base_xfail_png_path);
    free (xfail_path);
    free (diff_png_path);
    free (base_path);
    free (base_name);

    return ret;
}
Esempio n. 6
0
cairo_test_status_t
_cairo_test_context_run_for_target (cairo_test_context_t *ctx,
				    const cairo_boilerplate_target_t *target,
				    cairo_bool_t similar,
				    int dev_offset)
{
    cairo_test_status_t status;

    if (target->get_image_surface == NULL)
	return CAIRO_TEST_UNTESTED;

    if (similar && ! cairo_test_target_has_similar (ctx, target))
	return CAIRO_TEST_UNTESTED;

    cairo_test_log (ctx,
		    "Testing %s with %s%s target (dev offset %d)\n",
		    ctx->test_name,
		    similar ? " (similar) " : "",
		    target->name,
		    dev_offset);

    printf ("%s.%s.%s [%d]%s:\t", ctx->test_name, target->name,
	    cairo_boilerplate_content_name (target->content),
	    dev_offset,
	    similar ? " (similar)": "");
    fflush (stdout);

#if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
    if (! RUNNING_ON_VALGRIND) {
	void (* volatile old_segfault_handler)(int);
	void (* volatile old_segfpe_handler)(int);
	void (* volatile old_sigpipe_handler)(int);
	void (* volatile old_sigabrt_handler)(int);
	void (* volatile old_sigalrm_handler)(int);

	/* Set up a checkpoint to get back to in case of segfaults. */
#ifdef SIGSEGV
	old_segfault_handler = signal (SIGSEGV, segfault_handler);
#endif
#ifdef SIGFPE
	old_segfpe_handler = signal (SIGFPE, segfault_handler);
#endif
#ifdef SIGPIPE
	old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
#endif
#ifdef SIGABRT
	old_sigabrt_handler = signal (SIGABRT, segfault_handler);
#endif
#ifdef SIGALRM
	old_sigalrm_handler = signal (SIGALRM, segfault_handler);
#endif
	if (0 == setjmp (jmpbuf))
	    status = cairo_test_for_target (ctx, target, dev_offset, similar);
	else
	    status = CAIRO_TEST_CRASHED;
#ifdef SIGSEGV
	signal (SIGSEGV, old_segfault_handler);
#endif
#ifdef SIGFPE
	signal (SIGFPE, old_segfpe_handler);
#endif
#ifdef SIGPIPE
	signal (SIGPIPE, old_sigpipe_handler);
#endif
#ifdef SIGABRT
	signal (SIGABRT, old_sigabrt_handler);
#endif
#ifdef SIGALRM
	signal (SIGALRM, old_sigalrm_handler);
#endif
    } else {
	status = cairo_test_for_target (ctx, target, dev_offset, similar);
    }
#else
    status = cairo_test_for_target (ctx, target, dev_offset, similar);
#endif

    cairo_test_log (ctx,
		    "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
		    ctx->test_name, target->name,
		    cairo_boilerplate_content_name (target->content),
		    dev_offset, similar);
    switch (status) {
    case CAIRO_TEST_SUCCESS:
	printf ("PASS\n");
	cairo_test_log (ctx, "PASS\n");
	break;

    case CAIRO_TEST_UNTESTED:
	printf ("UNTESTED\n");
	cairo_test_log (ctx, "UNTESTED\n");
	break;

    default:
    case CAIRO_TEST_CRASHED:
	if (print_fail_on_stdout) {
	    printf ("!!!CRASHED!!!\n");
	} else {
	    /* eat the test name */
	    printf ("\r");
	    fflush (stdout);
	}
	cairo_test_log (ctx, "CRASHED\n");
	fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
		 ctx->test_name, target->name,
		 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
		 fail_face, normal_face);
	break;

    case CAIRO_TEST_ERROR:
	if (print_fail_on_stdout) {
	    printf ("!!!ERROR!!!\n");
	} else {
	    /* eat the test name */
	    printf ("\r");
	    fflush (stdout);
	}
	cairo_test_log (ctx, "ERROR\n");
	fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n",
		 ctx->test_name, target->name,
		 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
		 fail_face, normal_face);
	break;

    case CAIRO_TEST_XFAILURE:
	if (print_fail_on_stdout) {
	    printf ("XFAIL\n");
	} else {
	    /* eat the test name */
	    printf ("\r");
	    fflush (stdout);
	}
	fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n",
		 ctx->test_name, target->name,
		 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
		 xfail_face, normal_face);
	cairo_test_log (ctx, "XFAIL\n");
	break;

    case CAIRO_TEST_NEW:
	if (print_fail_on_stdout) {
	    printf ("NEW\n");
	} else {
	    /* eat the test name */
	    printf ("\r");
	    fflush (stdout);
	}
	fprintf (stderr, "%s.%s.%s [%d]%s:\t%sNEW%s\n",
		 ctx->test_name, target->name,
		 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
		 fail_face, normal_face);
	cairo_test_log (ctx, "NEW\n");
	break;

    case CAIRO_TEST_NO_MEMORY:
    case CAIRO_TEST_FAILURE:
	if (print_fail_on_stdout) {
	    printf ("FAIL\n");
	} else {
	    /* eat the test name */
	    printf ("\r");
	    fflush (stdout);
	}
	fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n",
		 ctx->test_name, target->name,
		 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
		 fail_face, normal_face);
	cairo_test_log (ctx, "FAIL\n");
	break;
    }
    fflush (stdout);

    return status;
}