PIPE_ALIGN_STACK static boolean test_one(unsigned verbose, FILE *fp, struct lp_type src_type, struct lp_type dst_type) { struct gallivm_state *gallivm; LLVMValueRef func = NULL; conv_test_ptr_t conv_test_ptr; boolean success; const unsigned n = LP_TEST_NUM_SAMPLES; int64_t cycles[LP_TEST_NUM_SAMPLES]; double cycles_avg = 0.0; unsigned num_srcs; unsigned num_dsts; double eps; unsigned i, j; if ((src_type.width >= dst_type.width && src_type.length > dst_type.length) || (src_type.width <= dst_type.width && src_type.length < dst_type.length)) { return TRUE; } /* Known failures * - fixed point 32 -> float 32 * - float 32 -> signed normalised integer 32 */ if ((src_type.floating && !dst_type.floating && dst_type.sign && dst_type.norm && src_type.width == dst_type.width) || (!src_type.floating && dst_type.floating && src_type.fixed && src_type.width == dst_type.width)) { return TRUE; } /* Known failures * - fixed point 32 -> float 32 * - float 32 -> signed normalised integer 32 */ if ((src_type.floating && !dst_type.floating && dst_type.sign && dst_type.norm && src_type.width == dst_type.width) || (!src_type.floating && dst_type.floating && src_type.fixed && src_type.width == dst_type.width)) { return TRUE; } if(verbose >= 1) dump_conv_types(stderr, src_type, dst_type); if (src_type.length > dst_type.length) { num_srcs = 1; num_dsts = src_type.length/dst_type.length; } else if (src_type.length < dst_type.length) { num_dsts = 1; num_srcs = dst_type.length/src_type.length; } else { num_dsts = 1; num_srcs = 1; } /* We must not loose or gain channels. Only precision */ assert(src_type.length * num_srcs == dst_type.length * num_dsts); eps = MAX2(lp_const_eps(src_type), lp_const_eps(dst_type)); gallivm = gallivm_create(); func = add_conv_test(gallivm, src_type, num_srcs, dst_type, num_dsts); gallivm_compile_module(gallivm); conv_test_ptr = (conv_test_ptr_t)gallivm_jit_function(gallivm, func); success = TRUE; for(i = 0; i < n && success; ++i) { unsigned src_stride = src_type.length*src_type.width/8; unsigned dst_stride = dst_type.length*dst_type.width/8; PIPE_ALIGN_VAR(LP_MIN_VECTOR_ALIGN) uint8_t src[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; PIPE_ALIGN_VAR(LP_MIN_VECTOR_ALIGN) uint8_t dst[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; double fref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; uint8_t ref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; int64_t start_counter = 0; int64_t end_counter = 0; for(j = 0; j < num_srcs; ++j) { random_vec(src_type, src + j*src_stride); read_vec(src_type, src + j*src_stride, fref + j*src_type.length); } for(j = 0; j < num_dsts; ++j) { write_vec(dst_type, ref + j*dst_stride, fref + j*dst_type.length); } start_counter = rdtsc(); conv_test_ptr(src, dst); end_counter = rdtsc(); cycles[i] = end_counter - start_counter; for(j = 0; j < num_dsts; ++j) { if(!compare_vec_with_eps(dst_type, dst + j*dst_stride, ref + j*dst_stride, eps)) success = FALSE; } if (!success || verbose >= 3) { if(verbose < 1) dump_conv_types(stderr, src_type, dst_type); if (success) { fprintf(stderr, "PASS\n"); } else { fprintf(stderr, "MISMATCH\n"); } for(j = 0; j < num_srcs; ++j) { fprintf(stderr, " Src%u: ", j); dump_vec(stderr, src_type, src + j*src_stride); fprintf(stderr, "\n"); } #if 1 fprintf(stderr, " Ref: "); for(j = 0; j < src_type.length*num_srcs; ++j) fprintf(stderr, " %f", fref[j]); fprintf(stderr, "\n"); #endif for(j = 0; j < num_dsts; ++j) { fprintf(stderr, " Dst%u: ", j); dump_vec(stderr, dst_type, dst + j*dst_stride); fprintf(stderr, "\n"); fprintf(stderr, " Ref%u: ", j); dump_vec(stderr, dst_type, ref + j*dst_stride); fprintf(stderr, "\n"); } } } /* * Unfortunately the output of cycle counter is not very reliable as it comes * -- sometimes we get outliers (due IRQs perhaps?) which are * better removed to avoid random or biased data. */ { double sum = 0.0, sum2 = 0.0; double avg, std; unsigned m; for(i = 0; i < n; ++i) { sum += cycles[i]; sum2 += cycles[i]*cycles[i]; } avg = sum/n; std = sqrtf((sum2 - n*avg*avg)/n); m = 0; sum = 0.0; for(i = 0; i < n; ++i) { if(fabs(cycles[i] - avg) <= 4.0*std) { sum += cycles[i]; ++m; } } cycles_avg = sum/m; } if(fp) write_tsv_row(fp, src_type, dst_type, cycles_avg, success); gallivm_free_function(gallivm, func, conv_test_ptr); gallivm_destroy(gallivm); return success; }
ALIGN_STACK static boolean test_one(unsigned verbose, FILE *fp, const struct pipe_blend_state *blend, enum vector_mode mode, struct lp_type type) { LLVMModuleRef module = NULL; LLVMValueRef func = NULL; LLVMExecutionEngineRef engine = NULL; LLVMModuleProviderRef provider = NULL; LLVMPassManagerRef pass = NULL; char *error = NULL; blend_test_ptr_t blend_test_ptr; boolean success; const unsigned n = LP_TEST_NUM_SAMPLES; int64_t cycles[LP_TEST_NUM_SAMPLES]; double cycles_avg = 0.0; unsigned i, j; if(verbose >= 1) dump_blend_type(stdout, blend, mode, type); module = LLVMModuleCreateWithName("test"); func = add_blend_test(module, blend, mode, type); if(LLVMVerifyModule(module, LLVMPrintMessageAction, &error)) { LLVMDumpModule(module); abort(); } LLVMDisposeMessage(error); provider = LLVMCreateModuleProviderForExistingModule(module); if (LLVMCreateJITCompiler(&engine, provider, 1, &error)) { if(verbose < 1) dump_blend_type(stderr, blend, mode, type); fprintf(stderr, "%s\n", error); LLVMDisposeMessage(error); abort(); } #if 0 pass = LLVMCreatePassManager(); LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), pass); /* These are the passes currently listed in llvm-c/Transforms/Scalar.h, * but there are more on SVN. */ LLVMAddConstantPropagationPass(pass); LLVMAddInstructionCombiningPass(pass); LLVMAddPromoteMemoryToRegisterPass(pass); LLVMAddGVNPass(pass); LLVMAddCFGSimplificationPass(pass); LLVMRunPassManager(pass, module); #else (void)pass; #endif if(verbose >= 2) LLVMDumpModule(module); blend_test_ptr = (blend_test_ptr_t)LLVMGetPointerToGlobal(engine, func); if(verbose >= 2) lp_disassemble(blend_test_ptr); success = TRUE; for(i = 0; i < n && success; ++i) { if(mode == AoS) { ALIGN16_ATTRIB uint8_t src[LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t dst[LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t con[LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t res[LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t ref[LP_NATIVE_VECTOR_WIDTH/8]; int64_t start_counter = 0; int64_t end_counter = 0; random_vec(type, src); random_vec(type, dst); random_vec(type, con); { double fsrc[LP_MAX_VECTOR_LENGTH]; double fdst[LP_MAX_VECTOR_LENGTH]; double fcon[LP_MAX_VECTOR_LENGTH]; double fref[LP_MAX_VECTOR_LENGTH]; read_vec(type, src, fsrc); read_vec(type, dst, fdst); read_vec(type, con, fcon); for(j = 0; j < type.length; j += 4) compute_blend_ref(blend, fsrc + j, fdst + j, fcon + j, fref + j); write_vec(type, ref, fref); } start_counter = rdtsc(); blend_test_ptr(src, dst, con, res); end_counter = rdtsc(); cycles[i] = end_counter - start_counter; if(!compare_vec(type, res, ref)) { success = FALSE; if(verbose < 1) dump_blend_type(stderr, blend, mode, type); fprintf(stderr, "MISMATCH\n"); fprintf(stderr, " Src: "); dump_vec(stderr, type, src); fprintf(stderr, "\n"); fprintf(stderr, " Dst: "); dump_vec(stderr, type, dst); fprintf(stderr, "\n"); fprintf(stderr, " Con: "); dump_vec(stderr, type, con); fprintf(stderr, "\n"); fprintf(stderr, " Res: "); dump_vec(stderr, type, res); fprintf(stderr, "\n"); fprintf(stderr, " Ref: "); dump_vec(stderr, type, ref); fprintf(stderr, "\n"); } } if(mode == SoA) { const unsigned stride = type.length*type.width/8; ALIGN16_ATTRIB uint8_t src[4*LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t dst[4*LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t con[4*LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t res[4*LP_NATIVE_VECTOR_WIDTH/8]; ALIGN16_ATTRIB uint8_t ref[4*LP_NATIVE_VECTOR_WIDTH/8]; int64_t start_counter = 0; int64_t end_counter = 0; boolean mismatch; for(j = 0; j < 4; ++j) { random_vec(type, src + j*stride); random_vec(type, dst + j*stride); random_vec(type, con + j*stride); } { double fsrc[4]; double fdst[4]; double fcon[4]; double fref[4]; unsigned k; for(k = 0; k < type.length; ++k) { for(j = 0; j < 4; ++j) { fsrc[j] = read_elem(type, src + j*stride, k); fdst[j] = read_elem(type, dst + j*stride, k); fcon[j] = read_elem(type, con + j*stride, k); } compute_blend_ref(blend, fsrc, fdst, fcon, fref); for(j = 0; j < 4; ++j) write_elem(type, ref + j*stride, k, fref[j]); } } start_counter = rdtsc(); blend_test_ptr(src, dst, con, res); end_counter = rdtsc(); cycles[i] = end_counter - start_counter; mismatch = FALSE; for (j = 0; j < 4; ++j) if(!compare_vec(type, res + j*stride, ref + j*stride)) mismatch = TRUE; if (mismatch) { success = FALSE; if(verbose < 1) dump_blend_type(stderr, blend, mode, type); fprintf(stderr, "MISMATCH\n"); for(j = 0; j < 4; ++j) { char channel = "RGBA"[j]; fprintf(stderr, " Src%c: ", channel); dump_vec(stderr, type, src + j*stride); fprintf(stderr, "\n"); fprintf(stderr, " Dst%c: ", channel); dump_vec(stderr, type, dst + j*stride); fprintf(stderr, "\n"); fprintf(stderr, " Con%c: ", channel); dump_vec(stderr, type, con + j*stride); fprintf(stderr, "\n"); fprintf(stderr, " Res%c: ", channel); dump_vec(stderr, type, res + j*stride); fprintf(stderr, "\n"); fprintf(stderr, " Ref%c: ", channel); dump_vec(stderr, type, ref + j*stride); fprintf(stderr, "\n"); } } } } /* * Unfortunately the output of cycle counter is not very reliable as it comes * -- sometimes we get outliers (due IRQs perhaps?) which are * better removed to avoid random or biased data. */ { double sum = 0.0, sum2 = 0.0; double avg, std; unsigned m; for(i = 0; i < n; ++i) { sum += cycles[i]; sum2 += cycles[i]*cycles[i]; } avg = sum/n; std = sqrtf((sum2 - n*avg*avg)/n); m = 0; sum = 0.0; for(i = 0; i < n; ++i) { if(fabs(cycles[i] - avg) <= 4.0*std) { sum += cycles[i]; ++m; } } cycles_avg = sum/m; } if(fp) write_tsv_row(fp, blend, mode, type, cycles_avg, success); if (!success) { if(verbose < 2) LLVMDumpModule(module); LLVMWriteBitcodeToFile(module, "blend.bc"); fprintf(stderr, "blend.bc written\n"); fprintf(stderr, "Invoke as \"llc -o - blend.bc\"\n"); abort(); } LLVMFreeMachineCodeForFunction(engine, func); LLVMDisposeExecutionEngine(engine); if(pass) LLVMDisposePassManager(pass); return success; }
PIPE_ALIGN_STACK static boolean test_one(unsigned verbose, FILE *fp, const struct pipe_blend_state *blend, struct lp_type type) { struct gallivm_state *gallivm; LLVMValueRef func = NULL; blend_test_ptr_t blend_test_ptr; boolean success; const unsigned n = LP_TEST_NUM_SAMPLES; int64_t cycles[LP_TEST_NUM_SAMPLES]; double cycles_avg = 0.0; unsigned i, j; const unsigned stride = lp_type_width(type)/8; if(verbose >= 1) dump_blend_type(stdout, blend, type); gallivm = gallivm_create(); func = add_blend_test(gallivm, blend, type); gallivm_compile_module(gallivm); blend_test_ptr = (blend_test_ptr_t)gallivm_jit_function(gallivm, func); success = TRUE; { uint8_t *src, *dst, *con, *res, *ref; src = align_malloc(stride, stride); dst = align_malloc(stride, stride); con = align_malloc(stride, stride); res = align_malloc(stride, stride); ref = align_malloc(stride, stride); for(i = 0; i < n && success; ++i) { int64_t start_counter = 0; int64_t end_counter = 0; random_vec(type, src); random_vec(type, dst); random_vec(type, con); { double fsrc[LP_MAX_VECTOR_LENGTH]; double fdst[LP_MAX_VECTOR_LENGTH]; double fcon[LP_MAX_VECTOR_LENGTH]; double fref[LP_MAX_VECTOR_LENGTH]; read_vec(type, src, fsrc); read_vec(type, dst, fdst); read_vec(type, con, fcon); for(j = 0; j < type.length; j += 4) compute_blend_ref(blend, fsrc + j, fdst + j, fcon + j, fref + j); write_vec(type, ref, fref); } start_counter = rdtsc(); blend_test_ptr(src, dst, con, res); end_counter = rdtsc(); cycles[i] = end_counter - start_counter; if(!compare_vec(type, res, ref)) { success = FALSE; if(verbose < 1) dump_blend_type(stderr, blend, type); fprintf(stderr, "MISMATCH\n"); fprintf(stderr, " Src: "); dump_vec(stderr, type, src); fprintf(stderr, "\n"); fprintf(stderr, " Dst: "); dump_vec(stderr, type, dst); fprintf(stderr, "\n"); fprintf(stderr, " Con: "); dump_vec(stderr, type, con); fprintf(stderr, "\n"); fprintf(stderr, " Res: "); dump_vec(stderr, type, res); fprintf(stderr, "\n"); fprintf(stderr, " Ref: "); dump_vec(stderr, type, ref); fprintf(stderr, "\n"); } } align_free(src); align_free(dst); align_free(con); align_free(res); align_free(ref); } /* * Unfortunately the output of cycle counter is not very reliable as it comes * -- sometimes we get outliers (due IRQs perhaps?) which are * better removed to avoid random or biased data. */ { double sum = 0.0, sum2 = 0.0; double avg, std; unsigned m; for(i = 0; i < n; ++i) { sum += cycles[i]; sum2 += cycles[i]*cycles[i]; } avg = sum/n; std = sqrtf((sum2 - n*avg*avg)/n); m = 0; sum = 0.0; for(i = 0; i < n; ++i) { if(fabs(cycles[i] - avg) <= 4.0*std) { sum += cycles[i]; ++m; } } cycles_avg = sum/m; } if(fp) write_tsv_row(fp, blend, type, cycles_avg, success); gallivm_free_function(gallivm, func, blend_test_ptr); gallivm_destroy(gallivm); return success; }
PIPE_ALIGN_STACK static boolean test_one(unsigned verbose, FILE *fp, struct lp_type src_type, struct lp_type dst_type) { LLVMModuleRef module = NULL; LLVMValueRef func = NULL; LLVMExecutionEngineRef engine = NULL; LLVMModuleProviderRef provider = NULL; LLVMPassManagerRef pass = NULL; char *error = NULL; conv_test_ptr_t conv_test_ptr; boolean success; const unsigned n = LP_TEST_NUM_SAMPLES; int64_t cycles[LP_TEST_NUM_SAMPLES]; double cycles_avg = 0.0; unsigned num_srcs; unsigned num_dsts; double eps; unsigned i, j; if(verbose >= 1) dump_conv_types(stdout, src_type, dst_type); if(src_type.length > dst_type.length) { num_srcs = 1; num_dsts = src_type.length/dst_type.length; } else { num_dsts = 1; num_srcs = dst_type.length/src_type.length; } assert(src_type.width * src_type.length == dst_type.width * dst_type.length); /* We must not loose or gain channels. Only precision */ assert(src_type.length * num_srcs == dst_type.length * num_dsts); eps = MAX2(lp_const_eps(src_type), lp_const_eps(dst_type)); module = LLVMModuleCreateWithName("test"); func = add_conv_test(module, src_type, num_srcs, dst_type, num_dsts); if(LLVMVerifyModule(module, LLVMPrintMessageAction, &error)) { LLVMDumpModule(module); abort(); } LLVMDisposeMessage(error); provider = LLVMCreateModuleProviderForExistingModule(module); if (LLVMCreateJITCompiler(&engine, provider, 1, &error)) { if(verbose < 1) dump_conv_types(stderr, src_type, dst_type); fprintf(stderr, "%s\n", error); LLVMDisposeMessage(error); abort(); } #if 0 pass = LLVMCreatePassManager(); LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), pass); /* These are the passes currently listed in llvm-c/Transforms/Scalar.h, * but there are more on SVN. */ LLVMAddConstantPropagationPass(pass); LLVMAddInstructionCombiningPass(pass); LLVMAddPromoteMemoryToRegisterPass(pass); LLVMAddGVNPass(pass); LLVMAddCFGSimplificationPass(pass); LLVMRunPassManager(pass, module); #else (void)pass; #endif if(verbose >= 2) LLVMDumpModule(module); conv_test_ptr = (conv_test_ptr_t)LLVMGetPointerToGlobal(engine, func); if(verbose >= 2) lp_disassemble(conv_test_ptr); success = TRUE; for(i = 0; i < n && success; ++i) { unsigned src_stride = src_type.length*src_type.width/8; unsigned dst_stride = dst_type.length*dst_type.width/8; PIPE_ALIGN_VAR(16) uint8_t src[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; PIPE_ALIGN_VAR(16) uint8_t dst[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; double fref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; uint8_t ref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH]; int64_t start_counter = 0; int64_t end_counter = 0; for(j = 0; j < num_srcs; ++j) { random_vec(src_type, src + j*src_stride); read_vec(src_type, src + j*src_stride, fref + j*src_type.length); } for(j = 0; j < num_dsts; ++j) { write_vec(dst_type, ref + j*dst_stride, fref + j*dst_type.length); } start_counter = rdtsc(); conv_test_ptr(src, dst); end_counter = rdtsc(); cycles[i] = end_counter - start_counter; for(j = 0; j < num_dsts; ++j) { if(!compare_vec_with_eps(dst_type, dst + j*dst_stride, ref + j*dst_stride, eps)) success = FALSE; } if (!success) { if(verbose < 1) dump_conv_types(stderr, src_type, dst_type); fprintf(stderr, "MISMATCH\n"); for(j = 0; j < num_srcs; ++j) { fprintf(stderr, " Src%u: ", j); dump_vec(stderr, src_type, src + j*src_stride); fprintf(stderr, "\n"); } #if 1 fprintf(stderr, " Ref: "); for(j = 0; j < src_type.length*num_srcs; ++j) fprintf(stderr, " %f", fref[j]); fprintf(stderr, "\n"); #endif for(j = 0; j < num_dsts; ++j) { fprintf(stderr, " Dst%u: ", j); dump_vec(stderr, dst_type, dst + j*dst_stride); fprintf(stderr, "\n"); fprintf(stderr, " Ref%u: ", j); dump_vec(stderr, dst_type, ref + j*dst_stride); fprintf(stderr, "\n"); } } } /* * Unfortunately the output of cycle counter is not very reliable as it comes * -- sometimes we get outliers (due IRQs perhaps?) which are * better removed to avoid random or biased data. */ { double sum = 0.0, sum2 = 0.0; double avg, std; unsigned m; for(i = 0; i < n; ++i) { sum += cycles[i]; sum2 += cycles[i]*cycles[i]; } avg = sum/n; std = sqrtf((sum2 - n*avg*avg)/n); m = 0; sum = 0.0; for(i = 0; i < n; ++i) { if(fabs(cycles[i] - avg) <= 4.0*std) { sum += cycles[i]; ++m; } } cycles_avg = sum/m; } if(fp) write_tsv_row(fp, src_type, dst_type, cycles_avg, success); if (!success) { static boolean firsttime = TRUE; if(firsttime) { if(verbose < 2) LLVMDumpModule(module); LLVMWriteBitcodeToFile(module, "conv.bc"); fprintf(stderr, "conv.bc written\n"); fprintf(stderr, "Invoke as \"llc -o - conv.bc\"\n"); firsttime = FALSE; /* abort(); */ } } LLVMFreeMachineCodeForFunction(engine, func); LLVMDisposeExecutionEngine(engine); if(pass) LLVMDisposePassManager(pass); return success; }