/* llmoduleprovider -> ExecutionEngine.t */
CAMLprim LLVMExecutionEngineRef
llvm_ee_create_fast_jit(LLVMModuleProviderRef MP) {
LLVMExecutionEngineRef JIT;
char *Error;
if (LLVMCreateJITCompiler(&JIT, MP, 1, &Error))
llvm_raise(llvm_ee_error_exn, Error);
return JIT;
}
SWIGEXPORT jint JNICALL Java_org_jllvm_bindings_ExecutionEngineJNI_LLVMCreateJITCompiler(JNIEnv *jenv, jclass jcls, jlong jarg1, jlong jarg2, jlong jarg3, jlong jarg4) {
jint jresult = 0 ;
LLVMExecutionEngineRef *arg1 = (LLVMExecutionEngineRef *) 0 ;
LLVMModuleProviderRef arg2 = (LLVMModuleProviderRef) 0 ;
unsigned int arg3 ;
char **arg4 = (char **) 0 ;
LLVMBool result;
(void)jenv;
(void)jcls;
arg1 = *(LLVMExecutionEngineRef **)&jarg1;
arg2 = *(LLVMModuleProviderRef *)&jarg2;
arg3 = (unsigned int)jarg3;
arg4 = *(char ***)&jarg4;
result = (LLVMBool)LLVMCreateJITCompiler(arg1,arg2,arg3,arg4);
jresult = (jint)result;
return jresult;
}
int LLVM_execute(LLVMCompiledProgram program)
{
LLVMModuleRef module = program.module;
LLVMValueRef function = program.function;
char *error = NULL; // Used to retrieve messages from functions
LLVMExecutionEngineRef engine;
LLVMModuleProviderRef provider = LLVMCreateModuleProviderForExistingModule(module);
error = NULL;
if(LLVMCreateJITCompiler(&engine, provider, 2, &error) != 0) {
fprintf(stderr, "%s\n", error);
LLVMDisposeMessage(error);
abort();
}
LLVMPassManagerRef pass = LLVMCreatePassManager();
LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), pass);
LLVMAddConstantPropagationPass(pass);
LLVMAddInstructionCombiningPass(pass);
LLVMAddPromoteMemoryToRegisterPass(pass);
LLVMAddGVNPass(pass);
LLVMAddCFGSimplificationPass(pass);
LLVMRunPassManager(pass, module);
#ifdef NDEBUG
#else
LLVMDumpModule(module);
#endif
LLVMGenericValueRef exec_args[] = {};
LLVMGenericValueRef exec_res = LLVMRunFunction(engine, function, 0, exec_args);
int result = LLVMGenericValueToInt(exec_res, 0);
LLVMDisposePassManager(pass);
LLVMDisposeExecutionEngine(engine);
return result;
}
/**
* Allocate gallivm LLVM objects.
* \return TRUE for success, FALSE for failure
*/
static boolean
init_gallivm_state(struct gallivm_state *gallivm)
{
assert(!gallivm->context);
assert(!gallivm->module);
assert(!gallivm->provider);
lp_build_init();
gallivm->context = LLVMContextCreate();
if (!gallivm->context)
goto fail;
gallivm->module = LLVMModuleCreateWithNameInContext("gallivm",
gallivm->context);
if (!gallivm->module)
goto fail;
gallivm->provider =
LLVMCreateModuleProviderForExistingModule(gallivm->module);
if (!gallivm->provider)
goto fail;
if (!GlobalEngine) {
/* We can only create one LLVMExecutionEngine (w/ LLVM 2.6 anyway) */
enum LLVM_CodeGenOpt_Level optlevel;
char *error = NULL;
if (gallivm_debug & GALLIVM_DEBUG_NO_OPT) {
optlevel = None;
}
else {
optlevel = Default;
}
if (LLVMCreateJITCompiler(&GlobalEngine, gallivm->provider,
(unsigned) optlevel, &error)) {
_debug_printf("%s\n", error);
LLVMDisposeMessage(error);
goto fail;
}
#if defined(DEBUG) || defined(PROFILE)
lp_register_oprofile_jit_event_listener(GlobalEngine);
#endif
}
gallivm->engine = GlobalEngine;
LLVMAddModuleProvider(gallivm->engine, gallivm->provider);//new
gallivm->target = LLVMGetExecutionEngineTargetData(gallivm->engine);
if (!gallivm->target)
goto fail;
if (!create_pass_manager(gallivm))
goto fail;
gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
if (!gallivm->builder)
goto fail;
return TRUE;
fail:
free_gallivm_state(gallivm);
return FALSE;
}
ALIGN_STACK
static boolean
test_one(unsigned verbose,
FILE *fp,
const struct pipe_blend_state *blend,
enum vector_mode mode,
struct lp_type type)
{
LLVMModuleRef module = NULL;
LLVMValueRef func = NULL;
LLVMExecutionEngineRef engine = NULL;
LLVMModuleProviderRef provider = NULL;
LLVMPassManagerRef pass = NULL;
char *error = NULL;
blend_test_ptr_t blend_test_ptr;
boolean success;
const unsigned n = LP_TEST_NUM_SAMPLES;
int64_t cycles[LP_TEST_NUM_SAMPLES];
double cycles_avg = 0.0;
unsigned i, j;
if(verbose >= 1)
dump_blend_type(stdout, blend, mode, type);
module = LLVMModuleCreateWithName("test");
func = add_blend_test(module, blend, mode, type);
if(LLVMVerifyModule(module, LLVMPrintMessageAction, &error)) {
LLVMDumpModule(module);
abort();
}
LLVMDisposeMessage(error);
provider = LLVMCreateModuleProviderForExistingModule(module);
if (LLVMCreateJITCompiler(&engine, provider, 1, &error)) {
if(verbose < 1)
dump_blend_type(stderr, blend, mode, type);
fprintf(stderr, "%s\n", error);
LLVMDisposeMessage(error);
abort();
}
#if 0
pass = LLVMCreatePassManager();
LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), pass);
/* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
* but there are more on SVN. */
LLVMAddConstantPropagationPass(pass);
LLVMAddInstructionCombiningPass(pass);
LLVMAddPromoteMemoryToRegisterPass(pass);
LLVMAddGVNPass(pass);
LLVMAddCFGSimplificationPass(pass);
LLVMRunPassManager(pass, module);
#else
(void)pass;
#endif
if(verbose >= 2)
LLVMDumpModule(module);
blend_test_ptr = (blend_test_ptr_t)LLVMGetPointerToGlobal(engine, func);
if(verbose >= 2)
lp_disassemble(blend_test_ptr);
success = TRUE;
for(i = 0; i < n && success; ++i) {
if(mode == AoS) {
ALIGN16_ATTRIB uint8_t src[LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t dst[LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t con[LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t res[LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t ref[LP_NATIVE_VECTOR_WIDTH/8];
int64_t start_counter = 0;
int64_t end_counter = 0;
random_vec(type, src);
random_vec(type, dst);
random_vec(type, con);
{
double fsrc[LP_MAX_VECTOR_LENGTH];
double fdst[LP_MAX_VECTOR_LENGTH];
double fcon[LP_MAX_VECTOR_LENGTH];
double fref[LP_MAX_VECTOR_LENGTH];
read_vec(type, src, fsrc);
read_vec(type, dst, fdst);
read_vec(type, con, fcon);
for(j = 0; j < type.length; j += 4)
compute_blend_ref(blend, fsrc + j, fdst + j, fcon + j, fref + j);
write_vec(type, ref, fref);
}
start_counter = rdtsc();
blend_test_ptr(src, dst, con, res);
end_counter = rdtsc();
cycles[i] = end_counter - start_counter;
if(!compare_vec(type, res, ref)) {
success = FALSE;
if(verbose < 1)
dump_blend_type(stderr, blend, mode, type);
fprintf(stderr, "MISMATCH\n");
fprintf(stderr, " Src: ");
dump_vec(stderr, type, src);
fprintf(stderr, "\n");
fprintf(stderr, " Dst: ");
dump_vec(stderr, type, dst);
fprintf(stderr, "\n");
fprintf(stderr, " Con: ");
dump_vec(stderr, type, con);
fprintf(stderr, "\n");
fprintf(stderr, " Res: ");
dump_vec(stderr, type, res);
fprintf(stderr, "\n");
fprintf(stderr, " Ref: ");
dump_vec(stderr, type, ref);
fprintf(stderr, "\n");
}
}
if(mode == SoA) {
const unsigned stride = type.length*type.width/8;
ALIGN16_ATTRIB uint8_t src[4*LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t dst[4*LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t con[4*LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t res[4*LP_NATIVE_VECTOR_WIDTH/8];
ALIGN16_ATTRIB uint8_t ref[4*LP_NATIVE_VECTOR_WIDTH/8];
int64_t start_counter = 0;
int64_t end_counter = 0;
boolean mismatch;
for(j = 0; j < 4; ++j) {
random_vec(type, src + j*stride);
random_vec(type, dst + j*stride);
random_vec(type, con + j*stride);
}
{
double fsrc[4];
double fdst[4];
double fcon[4];
double fref[4];
unsigned k;
for(k = 0; k < type.length; ++k) {
for(j = 0; j < 4; ++j) {
fsrc[j] = read_elem(type, src + j*stride, k);
fdst[j] = read_elem(type, dst + j*stride, k);
fcon[j] = read_elem(type, con + j*stride, k);
}
compute_blend_ref(blend, fsrc, fdst, fcon, fref);
for(j = 0; j < 4; ++j)
write_elem(type, ref + j*stride, k, fref[j]);
}
}
start_counter = rdtsc();
blend_test_ptr(src, dst, con, res);
end_counter = rdtsc();
cycles[i] = end_counter - start_counter;
mismatch = FALSE;
for (j = 0; j < 4; ++j)
if(!compare_vec(type, res + j*stride, ref + j*stride))
mismatch = TRUE;
if (mismatch) {
success = FALSE;
if(verbose < 1)
dump_blend_type(stderr, blend, mode, type);
fprintf(stderr, "MISMATCH\n");
for(j = 0; j < 4; ++j) {
char channel = "RGBA"[j];
fprintf(stderr, " Src%c: ", channel);
dump_vec(stderr, type, src + j*stride);
fprintf(stderr, "\n");
fprintf(stderr, " Dst%c: ", channel);
dump_vec(stderr, type, dst + j*stride);
fprintf(stderr, "\n");
fprintf(stderr, " Con%c: ", channel);
dump_vec(stderr, type, con + j*stride);
fprintf(stderr, "\n");
fprintf(stderr, " Res%c: ", channel);
dump_vec(stderr, type, res + j*stride);
fprintf(stderr, "\n");
fprintf(stderr, " Ref%c: ", channel);
dump_vec(stderr, type, ref + j*stride);
fprintf(stderr, "\n");
}
}
}
}
/*
* Unfortunately the output of cycle counter is not very reliable as it comes
* -- sometimes we get outliers (due IRQs perhaps?) which are
* better removed to avoid random or biased data.
*/
{
double sum = 0.0, sum2 = 0.0;
double avg, std;
unsigned m;
for(i = 0; i < n; ++i) {
sum += cycles[i];
sum2 += cycles[i]*cycles[i];
}
avg = sum/n;
std = sqrtf((sum2 - n*avg*avg)/n);
m = 0;
sum = 0.0;
for(i = 0; i < n; ++i) {
if(fabs(cycles[i] - avg) <= 4.0*std) {
sum += cycles[i];
++m;
}
}
cycles_avg = sum/m;
}
if(fp)
write_tsv_row(fp, blend, mode, type, cycles_avg, success);
if (!success) {
if(verbose < 2)
LLVMDumpModule(module);
LLVMWriteBitcodeToFile(module, "blend.bc");
fprintf(stderr, "blend.bc written\n");
fprintf(stderr, "Invoke as \"llc -o - blend.bc\"\n");
abort();
}
LLVMFreeMachineCodeForFunction(engine, func);
LLVMDisposeExecutionEngine(engine);
if(pass)
LLVMDisposePassManager(pass);
return success;
}
static boolean
init_gallivm_engine(struct gallivm_state *gallivm)
{
if (1) {
/* We can only create one LLVMExecutionEngine (w/ LLVM 2.6 anyway) */
enum LLVM_CodeGenOpt_Level optlevel;
char *error = NULL;
int ret;
if (gallivm_debug & GALLIVM_DEBUG_NO_OPT) {
optlevel = None;
}
else {
optlevel = Default;
}
#if USE_MCJIT
ret = lp_build_create_mcjit_compiler_for_module(&gallivm->engine,
gallivm->module,
(unsigned) optlevel,
&error);
#else
ret = LLVMCreateJITCompiler(&gallivm->engine, gallivm->provider,
(unsigned) optlevel, &error);
#endif
if (ret) {
_debug_printf("%s\n", error);
LLVMDisposeMessage(error);
goto fail;
}
#if defined(DEBUG) || defined(PROFILE)
lp_register_oprofile_jit_event_listener(gallivm->engine);
#endif
}
LLVMAddModuleProvider(gallivm->engine, gallivm->provider);//new
#if !USE_MCJIT
gallivm->target = LLVMGetExecutionEngineTargetData(gallivm->engine);
if (!gallivm->target)
goto fail;
#else
if (0) {
/*
* Dump the data layout strings.
*/
LLVMTargetDataRef target = LLVMGetExecutionEngineTargetData(gallivm->engine);
char *data_layout;
char *engine_data_layout;
data_layout = LLVMCopyStringRepOfTargetData(gallivm->target);
engine_data_layout = LLVMCopyStringRepOfTargetData(target);
if (1) {
debug_printf("module target data = %s\n", data_layout);
debug_printf("engine target data = %s\n", engine_data_layout);
}
free(data_layout);
free(engine_data_layout);
}
#endif
return TRUE;
fail:
return FALSE;
}
void
lp_build_init(void)
{
#ifdef DEBUG
gallivm_debug = debug_get_option_gallivm_debug();
#endif
lp_set_target_options();
LLVMInitializeNativeTarget();
LLVMLinkInJIT();
if (!lp_build_module)
lp_build_module = LLVMModuleCreateWithName("gallivm");
if (!lp_build_provider)
lp_build_provider = LLVMCreateModuleProviderForExistingModule(lp_build_module);
if (!lp_build_engine) {
enum LLVM_CodeGenOpt_Level optlevel;
char *error = NULL;
if (gallivm_debug & GALLIVM_DEBUG_NO_OPT) {
optlevel = None;
}
else {
optlevel = Default;
}
if (LLVMCreateJITCompiler(&lp_build_engine, lp_build_provider,
(unsigned)optlevel, &error)) {
_debug_printf("%s\n", error);
LLVMDisposeMessage(error);
assert(0);
}
#if defined(DEBUG) || defined(PROFILE)
lp_register_oprofile_jit_event_listener(lp_build_engine);
#endif
}
if (!lp_build_target)
lp_build_target = LLVMGetExecutionEngineTargetData(lp_build_engine);
if (!lp_build_pass) {
lp_build_pass = LLVMCreateFunctionPassManager(lp_build_provider);
LLVMAddTargetData(lp_build_target, lp_build_pass);
if ((gallivm_debug & GALLIVM_DEBUG_NO_OPT) == 0) {
/* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
* but there are more on SVN. */
/* TODO: Add more passes */
LLVMAddCFGSimplificationPass(lp_build_pass);
LLVMAddPromoteMemoryToRegisterPass(lp_build_pass);
LLVMAddConstantPropagationPass(lp_build_pass);
if(util_cpu_caps.has_sse4_1) {
/* FIXME: There is a bug in this pass, whereby the combination of fptosi
* and sitofp (necessary for trunc/floor/ceil/round implementation)
* somehow becomes invalid code.
*/
LLVMAddInstructionCombiningPass(lp_build_pass);
}
LLVMAddGVNPass(lp_build_pass);
} else {
/* We need at least this pass to prevent the backends to fail in
* unexpected ways.
*/
LLVMAddPromoteMemoryToRegisterPass(lp_build_pass);
}
}
util_cpu_detect();
#if 0
/* For simulating less capable machines */
util_cpu_caps.has_sse3 = 0;
util_cpu_caps.has_ssse3 = 0;
util_cpu_caps.has_sse4_1 = 0;
#endif
}
PIPE_ALIGN_STACK
static boolean
test_one(unsigned verbose,
FILE *fp,
struct lp_type src_type,
struct lp_type dst_type)
{
LLVMModuleRef module = NULL;
LLVMValueRef func = NULL;
LLVMExecutionEngineRef engine = NULL;
LLVMModuleProviderRef provider = NULL;
LLVMPassManagerRef pass = NULL;
char *error = NULL;
conv_test_ptr_t conv_test_ptr;
boolean success;
const unsigned n = LP_TEST_NUM_SAMPLES;
int64_t cycles[LP_TEST_NUM_SAMPLES];
double cycles_avg = 0.0;
unsigned num_srcs;
unsigned num_dsts;
double eps;
unsigned i, j;
if(verbose >= 1)
dump_conv_types(stdout, src_type, dst_type);
if(src_type.length > dst_type.length) {
num_srcs = 1;
num_dsts = src_type.length/dst_type.length;
}
else {
num_dsts = 1;
num_srcs = dst_type.length/src_type.length;
}
assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
/* We must not loose or gain channels. Only precision */
assert(src_type.length * num_srcs == dst_type.length * num_dsts);
eps = MAX2(lp_const_eps(src_type), lp_const_eps(dst_type));
module = LLVMModuleCreateWithName("test");
func = add_conv_test(module, src_type, num_srcs, dst_type, num_dsts);
if(LLVMVerifyModule(module, LLVMPrintMessageAction, &error)) {
LLVMDumpModule(module);
abort();
}
LLVMDisposeMessage(error);
provider = LLVMCreateModuleProviderForExistingModule(module);
if (LLVMCreateJITCompiler(&engine, provider, 1, &error)) {
if(verbose < 1)
dump_conv_types(stderr, src_type, dst_type);
fprintf(stderr, "%s\n", error);
LLVMDisposeMessage(error);
abort();
}
#if 0
pass = LLVMCreatePassManager();
LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), pass);
/* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
* but there are more on SVN. */
LLVMAddConstantPropagationPass(pass);
LLVMAddInstructionCombiningPass(pass);
LLVMAddPromoteMemoryToRegisterPass(pass);
LLVMAddGVNPass(pass);
LLVMAddCFGSimplificationPass(pass);
LLVMRunPassManager(pass, module);
#else
(void)pass;
#endif
if(verbose >= 2)
LLVMDumpModule(module);
conv_test_ptr = (conv_test_ptr_t)LLVMGetPointerToGlobal(engine, func);
if(verbose >= 2)
lp_disassemble(conv_test_ptr);
success = TRUE;
for(i = 0; i < n && success; ++i) {
unsigned src_stride = src_type.length*src_type.width/8;
unsigned dst_stride = dst_type.length*dst_type.width/8;
PIPE_ALIGN_VAR(16) uint8_t src[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
PIPE_ALIGN_VAR(16) uint8_t dst[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
double fref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
uint8_t ref[LP_MAX_VECTOR_LENGTH*LP_MAX_VECTOR_LENGTH];
int64_t start_counter = 0;
int64_t end_counter = 0;
for(j = 0; j < num_srcs; ++j) {
random_vec(src_type, src + j*src_stride);
read_vec(src_type, src + j*src_stride, fref + j*src_type.length);
}
for(j = 0; j < num_dsts; ++j) {
write_vec(dst_type, ref + j*dst_stride, fref + j*dst_type.length);
}
start_counter = rdtsc();
conv_test_ptr(src, dst);
end_counter = rdtsc();
cycles[i] = end_counter - start_counter;
for(j = 0; j < num_dsts; ++j) {
if(!compare_vec_with_eps(dst_type, dst + j*dst_stride, ref + j*dst_stride, eps))
success = FALSE;
}
if (!success) {
if(verbose < 1)
dump_conv_types(stderr, src_type, dst_type);
fprintf(stderr, "MISMATCH\n");
for(j = 0; j < num_srcs; ++j) {
fprintf(stderr, " Src%u: ", j);
dump_vec(stderr, src_type, src + j*src_stride);
fprintf(stderr, "\n");
}
#if 1
fprintf(stderr, " Ref: ");
for(j = 0; j < src_type.length*num_srcs; ++j)
fprintf(stderr, " %f", fref[j]);
fprintf(stderr, "\n");
#endif
for(j = 0; j < num_dsts; ++j) {
fprintf(stderr, " Dst%u: ", j);
dump_vec(stderr, dst_type, dst + j*dst_stride);
fprintf(stderr, "\n");
fprintf(stderr, " Ref%u: ", j);
dump_vec(stderr, dst_type, ref + j*dst_stride);
fprintf(stderr, "\n");
}
}
}
/*
* Unfortunately the output of cycle counter is not very reliable as it comes
* -- sometimes we get outliers (due IRQs perhaps?) which are
* better removed to avoid random or biased data.
*/
{
double sum = 0.0, sum2 = 0.0;
double avg, std;
unsigned m;
for(i = 0; i < n; ++i) {
sum += cycles[i];
sum2 += cycles[i]*cycles[i];
}
avg = sum/n;
std = sqrtf((sum2 - n*avg*avg)/n);
m = 0;
sum = 0.0;
for(i = 0; i < n; ++i) {
if(fabs(cycles[i] - avg) <= 4.0*std) {
sum += cycles[i];
++m;
}
}
cycles_avg = sum/m;
}
if(fp)
write_tsv_row(fp, src_type, dst_type, cycles_avg, success);
if (!success) {
static boolean firsttime = TRUE;
if(firsttime) {
if(verbose < 2)
LLVMDumpModule(module);
LLVMWriteBitcodeToFile(module, "conv.bc");
fprintf(stderr, "conv.bc written\n");
fprintf(stderr, "Invoke as \"llc -o - conv.bc\"\n");
firsttime = FALSE;
/* abort(); */
}
}
LLVMFreeMachineCodeForFunction(engine, func);
LLVMDisposeExecutionEngine(engine);
if(pass)
LLVMDisposePassManager(pass);
return success;
}
