/**
* Create new draw module context with gallivm state for LLVM JIT.
*/
struct draw_context *
draw_create_gallivm(struct pipe_context *pipe, struct gallivm_state *gallivm)
{
struct draw_context *draw = CALLOC_STRUCT( draw_context );
if (draw == NULL)
goto fail;
#if HAVE_LLVM
if (draw_get_option_use_llvm()) {
if (!gallivm) {
gallivm = gallivm_create();
draw->own_gallivm = gallivm;
}
if (gallivm)
draw->llvm = draw_llvm_create(draw, gallivm);
}
#endif
if (!draw_init(draw))
goto fail;
draw->pipe = pipe;
return draw;
fail:
draw_destroy( draw );
return NULL;
}
PIPE_ALIGN_STACK
static boolean
test_printf(unsigned verbose, FILE *fp,
const struct printf_test_case *testcase)
{
struct gallivm_state *gallivm;
LLVMValueRef test;
test_printf_t test_printf_func;
boolean success = TRUE;
gallivm = gallivm_create("test_module", LLVMGetGlobalContext());
test = add_printf_test(gallivm);
gallivm_compile_module(gallivm);
test_printf_func = (test_printf_t) gallivm_jit_function(gallivm, test);
gallivm_free_ir(gallivm);
test_printf_func(0);
gallivm_destroy(gallivm);
return success;
}
int main(int argc, char **argv)
{
unsigned verbose = 0;
FILE *fp = NULL;
unsigned long n = 1000;
unsigned i;
boolean success;
boolean single = FALSE;
struct gallivm_state *gallivm;
for(i = 1; i < argc; ++i) {
if(strcmp(argv[i], "-v") == 0)
++verbose;
else if(strcmp(argv[i], "-s") == 0)
single = TRUE;
else if(strcmp(argv[i], "-o") == 0)
fp = fopen(argv[++i], "wt");
else
n = atoi(argv[i]);
}
lp_build_init();
gallivm = gallivm_create();
util_cpu_detect();
if(fp) {
/* Warm up the caches */
test_some(gallivm, 0, NULL, 100);
write_tsv_header(fp);
}
if (single)
success = test_single(gallivm, verbose, fp);
else if (n)
success = test_some(gallivm, verbose, fp, n);
else
success = test_all(gallivm, verbose, fp);
if(fp)
fclose(fp);
return success ? 0 : 1;
}
PIPE_ALIGN_STACK
static boolean
test_one(unsigned verbose,
FILE *fp,
struct lp_type src_type,
struct lp_type dst_type)
{
struct gallivm_state *gallivm;
LLVMValueRef func = NULL;
conv_test_ptr_t conv_test_ptr;
boolean success;
const unsigned n = LP_TEST_NUM_SAMPLES;
int64_t cycles[LP_TEST_NUM_SAMPLES];
double cycles_avg = 0.0;
unsigned num_srcs;
unsigned num_dsts;
double eps;
unsigned i, j;
if ((src_type.width >= dst_type.width && src_type.length > dst_type.length) ||
(src_type.width <= dst_type.width && src_type.length < dst_type.length)) {
return TRUE;
}
/* Known failures
* - fixed point 32 -> float 32
* - float 32 -> signed normalised integer 32
*/
if ((src_type.floating && !dst_type.floating && dst_type.sign && dst_type.norm && src_type.width == dst_type.width) ||
(!src_type.floating && dst_type.floating && src_type.fixed && src_type.width == dst_type.width)) {
return TRUE;
}
/* Known failures
* - fixed point 32 -> float 32
* - float 32 -> signed normalised integer 32
*/
if ((src_type.floating && !dst_type.floating && dst_type.sign && dst_type.norm && src_type.width == dst_type.width) ||
(!src_type.floating && dst_type.floating && src_type.fixed && src_type.width == dst_type.width)) {
return TRUE;
}
if(verbose >= 1)
dump_conv_types(stderr, src_type, dst_type);
if (src_type.length > dst_type.length) {
num_srcs = 1;
num_dsts = src_type.length/dst_type.length;
}
else if (src_type.length < dst_type.length) {
num_dsts = 1;
num_srcs = dst_type.length/src_type.length;
}
else {
num_dsts = 1;
num_srcs = 1;
}
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
eps = MAX2(lp_const_eps(src_type), lp_const_eps(dst_type));
gallivm = gallivm_create();
func = add_conv_test(gallivm, src_type, num_srcs, dst_type, num_dsts);
gallivm_compile_module(gallivm);
conv_test_ptr = (conv_test_ptr_t)gallivm_jit_function(gallivm, func);
success = TRUE;
for(i = 0; i < n && success; ++i) {
unsigned src_stride = src_type.length*src_type.width/8;
unsigned dst_stride = dst_type.length*dst_type.width/8;
PIPE_ALIGN_VAR(LP_MIN_VECTOR_ALIGN) uint8_t src[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
PIPE_ALIGN_VAR(LP_MIN_VECTOR_ALIGN) uint8_t dst[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
double fref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
uint8_t ref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
int64_t start_counter = 0;
int64_t end_counter = 0;
for(j = 0; j < num_srcs; ++j) {
random_vec(src_type, src + j*src_stride);
read_vec(src_type, src + j*src_stride, fref + j*src_type.length);
}
for(j = 0; j < num_dsts; ++j) {
write_vec(dst_type, ref + j*dst_stride, fref + j*dst_type.length);
}
start_counter = rdtsc();
conv_test_ptr(src, dst);
end_counter = rdtsc();
cycles[i] = end_counter - start_counter;
for(j = 0; j < num_dsts; ++j) {
if(!compare_vec_with_eps(dst_type, dst + j*dst_stride, ref + j*dst_stride, eps))
success = FALSE;
}
if (!success || verbose >= 3) {
if(verbose < 1)
dump_conv_types(stderr, src_type, dst_type);
if (success) {
fprintf(stderr, "PASS\n");
}
else {
fprintf(stderr, "MISMATCH\n");
}
for(j = 0; j < num_srcs; ++j) {
fprintf(stderr, " Src%u: ", j);
dump_vec(stderr, src_type, src + j*src_stride);
fprintf(stderr, "\n");
}
#if 1
fprintf(stderr, " Ref: ");
for(j = 0; j < src_type.length*num_srcs; ++j)
fprintf(stderr, " %f", fref[j]);
fprintf(stderr, "\n");
#endif
for(j = 0; j < num_dsts; ++j) {
fprintf(stderr, " Dst%u: ", j);
dump_vec(stderr, dst_type, dst + j*dst_stride);
fprintf(stderr, "\n");
fprintf(stderr, " Ref%u: ", j);
dump_vec(stderr, dst_type, ref + j*dst_stride);
fprintf(stderr, "\n");
}
}
}
/*
* Unfortunately the output of cycle counter is not very reliable as it comes
* -- sometimes we get outliers (due IRQs perhaps?) which are
* better removed to avoid random or biased data.
*/
{
double sum = 0.0, sum2 = 0.0;
double avg, std;
unsigned m;
for(i = 0; i < n; ++i) {
sum += cycles[i];
sum2 += cycles[i]*cycles[i];
}
avg = sum/n;
std = sqrtf((sum2 - n*avg*avg)/n);
m = 0;
sum = 0.0;
for(i = 0; i < n; ++i) {
if(fabs(cycles[i] - avg) <= 4.0*std) {
sum += cycles[i];
++m;
}
}
cycles_avg = sum/m;
}
if(fp)
write_tsv_row(fp, src_type, dst_type, cycles_avg, success);
gallivm_free_function(gallivm, func, conv_test_ptr);
gallivm_destroy(gallivm);
return success;
}
PIPE_ALIGN_STACK
static boolean
test_one(unsigned verbose,
FILE *fp,
const struct pipe_blend_state *blend,
struct lp_type type)
{
struct gallivm_state *gallivm;
LLVMValueRef func = NULL;
blend_test_ptr_t blend_test_ptr;
boolean success;
const unsigned n = LP_TEST_NUM_SAMPLES;
int64_t cycles[LP_TEST_NUM_SAMPLES];
double cycles_avg = 0.0;
unsigned i, j;
const unsigned stride = lp_type_width(type)/8;
if(verbose >= 1)
dump_blend_type(stdout, blend, type);
gallivm = gallivm_create();
func = add_blend_test(gallivm, blend, type);
gallivm_compile_module(gallivm);
blend_test_ptr = (blend_test_ptr_t)gallivm_jit_function(gallivm, func);
success = TRUE;
{
uint8_t *src, *dst, *con, *res, *ref;
src = align_malloc(stride, stride);
dst = align_malloc(stride, stride);
con = align_malloc(stride, stride);
res = align_malloc(stride, stride);
ref = align_malloc(stride, stride);
for(i = 0; i < n && success; ++i) {
int64_t start_counter = 0;
int64_t end_counter = 0;
random_vec(type, src);
random_vec(type, dst);
random_vec(type, con);
{
double fsrc[LP_MAX_VECTOR_LENGTH];
double fdst[LP_MAX_VECTOR_LENGTH];
double fcon[LP_MAX_VECTOR_LENGTH];
double fref[LP_MAX_VECTOR_LENGTH];
read_vec(type, src, fsrc);
read_vec(type, dst, fdst);
read_vec(type, con, fcon);
for(j = 0; j < type.length; j += 4)
compute_blend_ref(blend, fsrc + j, fdst + j, fcon + j, fref + j);
write_vec(type, ref, fref);
}
start_counter = rdtsc();
blend_test_ptr(src, dst, con, res);
end_counter = rdtsc();
cycles[i] = end_counter - start_counter;
if(!compare_vec(type, res, ref)) {
success = FALSE;
if(verbose < 1)
dump_blend_type(stderr, blend, type);
fprintf(stderr, "MISMATCH\n");
fprintf(stderr, " Src: ");
dump_vec(stderr, type, src);
fprintf(stderr, "\n");
fprintf(stderr, " Dst: ");
dump_vec(stderr, type, dst);
fprintf(stderr, "\n");
fprintf(stderr, " Con: ");
dump_vec(stderr, type, con);
fprintf(stderr, "\n");
fprintf(stderr, " Res: ");
dump_vec(stderr, type, res);
fprintf(stderr, "\n");
fprintf(stderr, " Ref: ");
dump_vec(stderr, type, ref);
fprintf(stderr, "\n");
}
}
align_free(src);
align_free(dst);
align_free(con);
align_free(res);
align_free(ref);
}
/*
* Unfortunately the output of cycle counter is not very reliable as it comes
* -- sometimes we get outliers (due IRQs perhaps?) which are
* better removed to avoid random or biased data.
*/
{
double sum = 0.0, sum2 = 0.0;
double avg, std;
unsigned m;
for(i = 0; i < n; ++i) {
sum += cycles[i];
sum2 += cycles[i]*cycles[i];
}
avg = sum/n;
std = sqrtf((sum2 - n*avg*avg)/n);
m = 0;
sum = 0.0;
for(i = 0; i < n; ++i) {
if(fabs(cycles[i] - avg) <= 4.0*std) {
sum += cycles[i];
++m;
}
}
cycles_avg = sum/m;
}
if(fp)
write_tsv_row(fp, blend, type, cycles_avg, success);
gallivm_free_function(gallivm, func, blend_test_ptr);
gallivm_destroy(gallivm);
return success;
}
/*
* Test one LLVM unary arithmetic builder function.
*/
static boolean
test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test)
{
struct gallivm_state *gallivm;
LLVMValueRef test_func;
unary_func_t test_func_jit;
boolean success = TRUE;
int i, j;
int length = lp_native_vector_width / 32;
float *in, *out;
in = align_malloc(length * 4, length * 4);
out = align_malloc(length * 4, length * 4);
/* random NaNs or 0s could wreak havoc */
for (i = 0; i < length; i++) {
in[i] = 1.0;
}
gallivm = gallivm_create();
test_func = build_unary_test_func(gallivm, test);
gallivm_compile_module(gallivm);
test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_func);
for (j = 0; j < (test->num_values + length - 1) / length; j++) {
int num_vals = ((j + 1) * length <= test->num_values) ? length :
test->num_values % length;
for (i = 0; i < num_vals; ++i) {
in[i] = test->values[i+j*length];
}
test_func_jit(out, in);
for (i = 0; i < num_vals; ++i) {
float ref = test->ref(in[i]);
double error, precision;
bool pass;
if (util_inf_sign(ref) && util_inf_sign(out[i]) == util_inf_sign(ref)) {
error = 0;
} else {
error = fabs(out[i] - ref);
}
precision = error ? -log2(error/fabs(ref)) : FLT_MANT_DIG;
pass = precision >= test->precision;
if (isnan(ref)) {
continue;
}
if (!pass || verbose) {
printf("%s(%.9g): ref = %.9g, out = %.9g, precision = %f bits, %s\n",
test->name, in[i], ref, out[i], precision,
pass ? "PASS" : "FAIL");
}
if (!pass) {
success = FALSE;
}
}
}
gallivm_free_function(gallivm, test_func, test_func_jit);
gallivm_destroy(gallivm);
align_free(in);
align_free(out);
return success;
}