LLVMBasicBlockRef JITImpl::getOrCreateMemoryCheckBailoutBlock(unsigned index)
{
if (index == 0) {
if (interpretOneBB) {
return interpretOneBB;
}
} else if (endTraceBB) {
return endTraceBB;
}
LLVMBasicBlockRef savedInsertPoint = LLVMGetInsertBlock(builder);
LLVMBasicBlockRef bailoutBB = LLVMAppendBasicBlock(getCurrentFunction(), "");
LLVMPositionBuilderAtEnd(builder, bailoutBB);
if (index == 0) {
LLVMValueRef args[] = {
threadParam
};
LLVMValueRef call = emitCallToBeInlined(functions.jitInterpretOne, args, 1);
LLVMBuildRet(builder, call);
interpretOneBB = bailoutBB;
} else {
ensureEarlyReturnBB(LLVMGetReturnType(jitFunctionType));
earlyReturnIncomingValues.push_back(
LLVMConstInt(LLVMGetReturnType(jitFunctionType),
JIT_RETURN_END_TRACE, false));
earlyReturnIncomingBlocks.push_back(LLVMGetInsertBlock(builder));
LLVMBuildBr(builder, earlyReturnBB);
endTraceBB = bailoutBB;
}
LLVMPositionBuilderAtEnd(builder, savedInsertPoint);
return bailoutBB;
}
void JITImpl::emitCondEarlyReturn(LLVMValueRef cond, LLVMValueRef retval)
{
ensureEarlyReturnBB(LLVMGetReturnType((jitFunctionType)));
earlyReturnIncomingValues.push_back(retval);
earlyReturnIncomingBlocks.push_back(LLVMGetInsertBlock(builder));
emitCondBrToBlock(cond, earlyReturnBB);
}
LLVMValueRef gen_return(compile_t* c, ast_t* ast)
{
ast_t* expr = ast_child(ast);
LLVMValueRef value = gen_expr(c, expr);
size_t clause;
ast_t* try_expr = ast_try_clause(ast, &clause);
// Do the then block only if we return in the body or else clause.
// In the then block, return without doing the then block.
if((try_expr != NULL) && (clause != 2))
gen_expr(c, ast_childidx(try_expr, 2));
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(codegen_fun(c)));
LLVMTypeRef r_type = LLVMGetReturnType(f_type);
codegen_debugloc(c, ast);
if(LLVMGetTypeKind(r_type) != LLVMVoidTypeKind)
{
LLVMValueRef ret = gen_assign_cast(c, r_type, value, ast_type(expr));
codegen_scope_lifetime_end(c);
LLVMBuildRet(c->builder, ret);
} else {
codegen_scope_lifetime_end(c);
LLVMBuildRetVoid(c->builder);
}
codegen_debugloc(c, NULL);
return GEN_NOVALUE;
}
static LLVMValueRef make_unbox_function(compile_t* c, gentype_t* g,
const char* name)
{
LLVMValueRef fun = LLVMGetNamedFunction(c->module, name);
if(fun == NULL)
return LLVMConstNull(c->void_ptr);
// Create a new unboxing function that forwards to the real function.
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(fun));
int count = LLVMCountParamTypes(f_type);
// If it takes no arguments, it's a special number constructor. Don't put it
// in the vtable.
if(count == 0)
return LLVMConstNull(c->void_ptr);
size_t buf_size = count *sizeof(LLVMTypeRef);
LLVMTypeRef* params = (LLVMTypeRef*)pool_alloc_size(buf_size);
LLVMGetParamTypes(f_type, params);
LLVMTypeRef ret_type = LLVMGetReturnType(f_type);
// It's the same type, but it takes the boxed type instead of the primitive
// type as the receiver.
params[0] = g->structure_ptr;
const char* unbox_name = genname_unbox(name);
LLVMTypeRef unbox_type = LLVMFunctionType(ret_type, params, count, false);
LLVMValueRef unbox_fun = codegen_addfun(c, unbox_name, unbox_type);
codegen_startfun(c, unbox_fun, false);
// Extract the primitive type from element 1 and call the real function.
LLVMValueRef this_ptr = LLVMGetParam(unbox_fun, 0);
LLVMValueRef primitive_ptr = LLVMBuildStructGEP(c->builder, this_ptr, 1, "");
LLVMValueRef primitive = LLVMBuildLoad(c->builder, primitive_ptr, "");
LLVMValueRef* args = (LLVMValueRef*)pool_alloc_size(buf_size);
args[0] = primitive;
for(int i = 1; i < count; i++)
args[i] = LLVMGetParam(unbox_fun, i);
LLVMValueRef result = codegen_call(c, fun, args, count);
LLVMBuildRet(c->builder, result);
codegen_finishfun(c);
pool_free_size(buf_size, params);
pool_free_size(buf_size, args);
return LLVMConstBitCast(unbox_fun, c->void_ptr);
}
static bool genfun_fun(compile_t* c, reachable_type_t* t,
reachable_method_t* m)
{
assert(m->func != NULL);
AST_GET_CHILDREN(m->r_fun, cap, id, typeparams, params, result, can_error,
body);
if(m->name == c->str__final)
{
t->final_fn = m->func;
LLVMSetFunctionCallConv(m->func, LLVMCCallConv);
}
codegen_startfun(c, m->func, m->di_file, m->di_method);
name_params(c, t, m, params, m->func);
LLVMValueRef value = gen_expr(c, body);
if(value == NULL)
return false;
if(value != GEN_NOVALUE)
{
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(m->func));
LLVMTypeRef r_type = LLVMGetReturnType(f_type);
// If the result type is known to be a tuple, do the correct assignment
// cast even if the body type is not a tuple.
ast_t* body_type = ast_type(body);
if(ast_id(result) == TK_TUPLETYPE)
body_type = result;
LLVMValueRef ret = gen_assign_cast(c, r_type, value, body_type);
if(ret == NULL)
return false;
codegen_debugloc(c, ast_childlast(body));
LLVMBuildRet(c->builder, ret);
codegen_debugloc(c, NULL);
}
codegen_finishfun(c);
return true;
}
static LLVMValueRef genfun_fun(compile_t* c, gentype_t* g, const char *name,
ast_t* typeargs)
{
ast_t* fun = get_fun(g, name, typeargs);
LLVMValueRef func = get_prototype(c, g, name, typeargs, fun);
if(func == NULL)
{
ast_free_unattached(fun);
return NULL;
}
if(LLVMCountBasicBlocks(func) != 0)
{
ast_free_unattached(fun);
return func;
}
codegen_startfun(c, func, ast_debug(fun));
name_params(c, g->ast, ast_childidx(fun, 3), func);
genfun_dwarf(c, g, name, typeargs, fun);
ast_t* body = ast_childidx(fun, 6);
LLVMValueRef value = gen_expr(c, body);
if(value == NULL)
{
ast_free_unattached(fun);
return NULL;
} else if(value != GEN_NOVALUE) {
genfun_dwarf_return(c, body);
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(func));
LLVMTypeRef r_type = LLVMGetReturnType(f_type);
LLVMValueRef ret = gen_assign_cast(c, r_type, value, ast_type(body));
LLVMBuildRet(c->builder, ret);
}
codegen_finishfun(c);
ast_free_unattached(fun);
return func;
}
static LLVMValueRef make_unbox_function(compile_t* c, reach_type_t* t,
reach_method_t* m)
{
// Create a new unboxing function that forwards to the real function.
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(m->func));
int count = LLVMCountParamTypes(f_type);
// Leave space for a receiver if it's a constructor vtable entry.
size_t buf_size = (count + 1) * sizeof(LLVMTypeRef);
LLVMTypeRef* params = (LLVMTypeRef*)ponyint_pool_alloc_size(buf_size);
LLVMGetParamTypes(f_type, params);
LLVMTypeRef ret_type = LLVMGetReturnType(f_type);
const char* unbox_name = genname_unbox(m->full_name);
if(ast_id(m->r_fun) != TK_NEW)
{
// It's the same type, but it takes the boxed type instead of the primitive
// type as the receiver.
params[0] = t->structure_ptr;
} else {
// For a constructor, the unbox_fun has a receiver, even though the real
// method does not.
memmove(¶ms[1], ¶ms[0], count * sizeof(LLVMTypeRef*));
params[0] = t->structure_ptr;
count++;
}
LLVMTypeRef unbox_type = LLVMFunctionType(ret_type, params, count, false);
LLVMValueRef unbox_fun = codegen_addfun(c, unbox_name, unbox_type);
codegen_startfun(c, unbox_fun, NULL, NULL);
// Extract the primitive type from element 1 and call the real function.
LLVMValueRef this_ptr = LLVMGetParam(unbox_fun, 0);
LLVMValueRef primitive_ptr = LLVMBuildStructGEP(c->builder, this_ptr, 1, "");
LLVMValueRef primitive = LLVMBuildLoad(c->builder, primitive_ptr, "");
LLVMValueRef* args = (LLVMValueRef*)ponyint_pool_alloc_size(buf_size);
if(ast_id(m->r_fun) != TK_NEW)
{
// If it's not a constructor, pass the extracted primitive as the receiver.
args[0] = primitive;
for(int i = 1; i < count; i++)
args[i] = LLVMGetParam(unbox_fun, i);
} else {
count--;
for(int i = 0; i < count; i++)
args[i] = LLVMGetParam(unbox_fun, i + 1);
}
LLVMValueRef result = codegen_call(c, m->func, args, count);
LLVMBuildRet(c->builder, result);
codegen_finishfun(c);
ponyint_pool_free_size(buf_size, params);
ponyint_pool_free_size(buf_size, args);
return LLVMConstBitCast(unbox_fun, c->void_ptr);
}
/// Try and compile a fragment starting at the specified address. Returns
/// true if successful setting \a nextAddress to the first instruction after
/// the fragment. If unsuccessful returns false and sets \a nextAddress to the
/// address after the current function. \a endOfBlock is set to true if the
/// next address is in a new basic block.
bool JITImpl::
compileOneFragment(Core &core, JITCoreInfo &coreInfo, uint32_t startPc,
bool &endOfBlock, uint32_t &pcAfterFragment)
{
assert(initialized);
resetPerFunctionState();
std::map<uint32_t,JITFunctionInfo*>::iterator infoIt =
coreInfo.functionMap.find(startPc);
JITFunctionInfo *info =
(infoIt == coreInfo.functionMap.end()) ? 0 : infoIt->second;
if (info && !info->isStub) {
endOfBlock = true;
return false;
}
std::vector<InstructionOpcode> opcode;
std::vector<Operands> operands;
if (!getFragmentToCompile(core, startPc, opcode, operands,
endOfBlock, pcAfterFragment)) {
return false;
}
std::queue<std::pair<uint32_t,MemoryCheck*> > checks;
placeMemoryChecks(opcode, operands, checks);
LLVMValueRef f;
if (info) {
f = info->value;
info->func = 0;
info->isStub = false;
deleteFunctionBody(f);
} else {
info = new JITFunctionInfo(startPc);
coreInfo.functionMap.insert(std::make_pair(startPc, info));
// Create function to contain the code we are about to add.
info->value = f = LLVMAddFunction(module, "", jitFunctionType);
LLVMSetFunctionCallConv(f, LLVMFastCallConv);
}
threadParam = LLVMGetParam(f, 0);
LLVMValueRef ramBase = LLVMConstInt(LLVMInt32Type(), core.ram_base, false);
ramSizeLog2Param = LLVMConstInt(LLVMInt32Type(), core.ramSizeLog2, false);
LLVMBasicBlockRef entryBB = LLVMAppendBasicBlock(f, "entry");
LLVMPositionBuilderAtEnd(builder, entryBB);
uint32_t pc = startPc;
bool needsReturn = true;
for (unsigned i = 0, e = opcode.size(); i != e; ++i) {
InstructionOpcode opc = opcode[i];
const Operands &ops = operands[i];
InstructionProperties *properties = &instructionProperties[opc];
uint32_t nextPc = pc + properties->size / 2;
emitMemoryChecks(i, checks);
// Lookup function to call.
LLVMValueRef callee = LLVMGetNamedFunction(module, properties->function);
assert(callee && "Function for instruction not found in module");
LLVMTypeRef calleeType = LLVMGetElementType(LLVMTypeOf(callee));
const unsigned fixedArgs = 4;
const unsigned maxOperands = 6;
unsigned numArgs = properties->getNumExplicitOperands() + fixedArgs;
assert(LLVMCountParamTypes(calleeType) == numArgs);
LLVMTypeRef paramTypes[fixedArgs + maxOperands];
assert(numArgs <= (fixedArgs + maxOperands));
LLVMGetParamTypes(calleeType, paramTypes);
// Build call.
LLVMValueRef args[fixedArgs + maxOperands];
args[0] = threadParam;
args[1] = LLVMConstInt(paramTypes[1], nextPc, false);
args[2] = ramBase;
args[3] = ramSizeLog2Param;
for (unsigned i = fixedArgs; i < numArgs; i++) {
uint32_t value =
properties->getNumExplicitOperands() <= 3 ? ops.ops[i - fixedArgs] :
ops.lops[i - fixedArgs];
args[i] = LLVMConstInt(paramTypes[i], value, false);
}
LLVMValueRef call = emitCallToBeInlined(callee, args, numArgs);
checkReturnValue(call, *properties);
if (properties->mayBranch() && properties->function &&
emitJumpToNextFragment(opc, ops, coreInfo, nextPc, info)) {
needsReturn = false;
}
pc = nextPc;
}
assert(checks.empty() && "Not all checks emitted");
if (needsReturn) {
LLVMValueRef args[] = {
threadParam
};
emitCallToBeInlined(functions.jitUpdateExecutionFrequency, args, 1);
// Build return.
LLVMBuildRet(builder,
LLVMConstInt(LLVMGetReturnType(jitFunctionType),
JIT_RETURN_CONTINUE, 0));
}
// Add incoming phi values.
if (earlyReturnBB) {
LLVMAddIncoming(earlyReturnPhi, &earlyReturnIncomingValues[0],
&earlyReturnIncomingBlocks[0],
earlyReturnIncomingValues.size());
}
if (DEBUG_JIT) {
LLVMDumpValue(f);
LLVMVerifyFunction(f, LLVMAbortProcessAction);
}
// Optimize.
for (std::vector<LLVMValueRef>::iterator it = calls.begin(), e = calls.end();
it != e; ++it) {
LLVMExtraInlineFunction(*it);
}
LLVMRunFunctionPassManager(FPM, f);
if (DEBUG_JIT) {
LLVMDumpValue(f);
}
// Compile.
JITInstructionFunction_t compiledFunction =
reinterpret_cast<JITInstructionFunction_t>(
LLVMRecompileAndRelinkFunction(executionEngine, f));
info->isStub = false;
info->func = compiledFunction;
core.setOpcode(startPc, getFunctionThunk(*info), (pc - startPc) * 2);
return true;
}
LLVMValueRef gen_ffi(compile_t* c, ast_t* ast)
{
AST_GET_CHILDREN(ast, id, typeargs, args, named_args, can_err);
bool err = (ast_id(can_err) == TK_QUESTION);
// Get the function name, +1 to skip leading @
const char* f_name = ast_name(id) + 1;
deferred_reification_t* reify = c->frame->reify;
// Get the return type.
ast_t* type = deferred_reify(reify, ast_type(ast), c->opt);
reach_type_t* t = reach_type(c->reach, type);
pony_assert(t != NULL);
ast_free_unattached(type);
// Get the function. First check if the name is in use by a global and error
// if it's the case.
ffi_decl_t* ffi_decl;
bool is_func = false;
LLVMValueRef func = LLVMGetNamedGlobal(c->module, f_name);
if(func == NULL)
{
func = LLVMGetNamedFunction(c->module, f_name);
is_func = true;
}
if(func == NULL)
{
// If we have no prototype, declare one.
ast_t* decl = (ast_t*)ast_data(ast);
if(decl != NULL)
{
// Define using the declared types.
AST_GET_CHILDREN(decl, decl_id, decl_ret, decl_params, decl_err);
err = (ast_id(decl_err) == TK_QUESTION);
func = declare_ffi(c, f_name, t, decl_params, false);
} else if(!strncmp(f_name, "llvm.", 5) || !strncmp(f_name, "internal.", 9)) {
// Intrinsic, so use the exact types we supply.
func = declare_ffi(c, f_name, t, args, true);
} else {
// Make it varargs.
func = declare_ffi_vararg(c, f_name, t);
}
size_t index = HASHMAP_UNKNOWN;
#ifndef PONY_NDEBUG
ffi_decl_t k;
k.func = func;
ffi_decl = ffi_decls_get(&c->ffi_decls, &k, &index);
pony_assert(ffi_decl == NULL);
#endif
ffi_decl = POOL_ALLOC(ffi_decl_t);
ffi_decl->func = func;
ffi_decl->decl = (decl != NULL) ? decl : ast;
ffi_decls_putindex(&c->ffi_decls, ffi_decl, index);
} else {
ffi_decl_t k;
k.func = func;
size_t index = HASHMAP_UNKNOWN;
ffi_decl = ffi_decls_get(&c->ffi_decls, &k, &index);
if((ffi_decl == NULL) && (!is_func || LLVMHasMetadataStr(func, "pony.abi")))
{
ast_error(c->opt->check.errors, ast, "cannot use '%s' as an FFI name: "
"name is already in use by the internal ABI", f_name);
return NULL;
}
pony_assert(is_func);
}
// Generate the arguments.
int count = (int)ast_childcount(args);
size_t buf_size = count * sizeof(LLVMValueRef);
LLVMValueRef* f_args = (LLVMValueRef*)ponyint_pool_alloc_size(buf_size);
LLVMTypeRef f_type = LLVMGetElementType(LLVMTypeOf(func));
LLVMTypeRef* f_params = NULL;
bool vararg = (LLVMIsFunctionVarArg(f_type) != 0);
if(!vararg)
{
if(count != (int)LLVMCountParamTypes(f_type))
{
ast_error(c->opt->check.errors, ast,
"conflicting declarations for FFI function: declarations have an "
"incompatible number of parameters");
if(ffi_decl != NULL)
ast_error_continue(c->opt->check.errors, ffi_decl->decl, "first "
"declaration is here");
return NULL;
}
f_params = (LLVMTypeRef*)ponyint_pool_alloc_size(buf_size);
LLVMGetParamTypes(f_type, f_params);
}
ast_t* arg = ast_child(args);
for(int i = 0; i < count; i++)
{
f_args[i] = gen_expr(c, arg);
if(!vararg)
f_args[i] = cast_ffi_arg(c, ffi_decl, ast, f_args[i], f_params[i],
"parameters");
if(f_args[i] == NULL)
{
ponyint_pool_free_size(buf_size, f_args);
return NULL;
}
arg = ast_sibling(arg);
}
// If we can error out and we have an invoke target, generate an invoke
// instead of a call.
LLVMValueRef result;
codegen_debugloc(c, ast);
if(err && (c->frame->invoke_target != NULL))
result = invoke_fun(c, func, f_args, count, "", false);
else
result = LLVMBuildCall(c->builder, func, f_args, count, "");
codegen_debugloc(c, NULL);
ponyint_pool_free_size(buf_size, f_args);
if(!vararg)
ponyint_pool_free_size(buf_size, f_params);
compile_type_t* c_t = (compile_type_t*)t->c_type;
// Special case a None return value, which is used for void functions.
bool isnone = is_none(t->ast);
bool isvoid = LLVMGetReturnType(f_type) == c->void_type;
if(isnone && isvoid)
{
result = c_t->instance;
} else if(isnone != isvoid) {
report_ffi_type_err(c, ffi_decl, ast, "return values");
return NULL;
}
result = cast_ffi_arg(c, ffi_decl, ast, result, c_t->use_type,
"return values");
result = gen_assign_cast(c, c_t->use_type, result, t->ast_cap);
return result;
}
