llvm::Value *
Target::SizeOf(LLVM_TYPE_CONST llvm::Type *type,
llvm::BasicBlock *insertAtEnd) {
if (isa == Target::GENERIC &&
lGenericTypeLayoutIndeterminate(type)) {
llvm::Value *index[1] = { LLVMInt32(1) };
LLVM_TYPE_CONST llvm::PointerType *ptrType = llvm::PointerType::get(type, 0);
llvm::Value *voidPtr = llvm::ConstantPointerNull::get(ptrType);
#if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn)
llvm::ArrayRef<llvm::Value *> arrayRef(&index[0], &index[1]);
llvm::Instruction *gep =
llvm::GetElementPtrInst::Create(voidPtr, arrayRef, "sizeof_gep",
insertAtEnd);
#else
llvm::Instruction *gep =
llvm::GetElementPtrInst::Create(voidPtr, &index[0], &index[1],
"sizeof_gep", insertAtEnd);
#endif
if (is32Bit || g->opt.force32BitAddressing)
return new llvm::PtrToIntInst(gep, LLVMTypes::Int32Type,
"sizeof_int", insertAtEnd);
else
return new llvm::PtrToIntInst(gep, LLVMTypes::Int64Type,
"sizeof_int", insertAtEnd);
}
const llvm::TargetData *td = GetTargetMachine()->getTargetData();
Assert(td != NULL);
uint64_t byteSize = td->getTypeSizeInBits(type) / 8;
if (is32Bit || g->opt.force32BitAddressing)
return LLVMInt32((int32_t)byteSize);
else
return LLVMInt64(byteSize);
}
/** Utility routine that defines a constant int32 with given value, adding
the symbol to both the ispc symbol table and the given LLVM module.
*/
static void
lDefineConstantInt(const char *name, int val, llvm::Module *module,
SymbolTable *symbolTable) {
Symbol *sym =
new Symbol(name, SourcePos(), AtomicType::UniformInt32->GetAsConstType(),
SC_STATIC);
sym->constValue = new ConstExpr(sym->type, val, SourcePos());
llvm::Type *ltype = LLVMTypes::Int32Type;
llvm::Constant *linit = LLVMInt32(val);
// Use WeakODRLinkage rather than InternalLinkage so that a definition
// survives even if it's not used in the module, so that the symbol is
// there in the debugger.
llvm::GlobalValue::LinkageTypes linkage = g->generateDebuggingSymbols ?
llvm::GlobalValue::WeakODRLinkage : llvm::GlobalValue::InternalLinkage;
sym->storagePtr = new llvm::GlobalVariable(*module, ltype, true, linkage,
linit, name);
symbolTable->AddVariable(sym);
if (m->diBuilder != NULL) {
llvm::DIFile file;
llvm::DIType diType = sym->type->GetDIType(file);
Assert(diType.Verify());
// FIXME? DWARF says that this (and programIndex below) should
// have the DW_AT_artifical attribute. It's not clear if this
// matters for anything though.
llvm::DIGlobalVariable var =
m->diBuilder->createGlobalVariable(name,
file,
0 /* line */,
diType,
true /* static */,
sym->storagePtr);
Assert(var.Verify());
}
}
llvm::Constant *
LLVMInt32Vector(const int32_t *ivec) {
std::vector<llvm::Constant *> vals;
for (int i = 0; i < g->target.vectorWidth; ++i)
vals.push_back(LLVMInt32(ivec[i]));
return llvm::ConstantVector::get(vals);
}
llvm::Constant *
LLVMInt32Vector(int32_t ival) {
llvm::Constant *v = LLVMInt32(ival);
std::vector<llvm::Constant *> vals;
for (int i = 0; i < g->target.vectorWidth; ++i)
vals.push_back(v);
return llvm::ConstantVector::get(vals);
}
llvm::Value *
Target::SizeOf(LLVM_TYPE_CONST llvm::Type *type) {
const llvm::TargetData *td = GetTargetMachine()->getTargetData();
Assert(td != NULL);
uint64_t byteSize = td->getTypeSizeInBits(type) / 8;
if (is32Bit || g->opt.force32BitAddressing)
return LLVMInt32((int32_t)byteSize);
else
return LLVMInt64(byteSize);
}
llvm::Value *
Target::StructOffset(LLVM_TYPE_CONST llvm::Type *type, int element,
llvm::BasicBlock *insertAtEnd) {
if (isa == Target::GENERIC &&
lGenericTypeLayoutIndeterminate(type) == true) {
llvm::Value *indices[2] = { LLVMInt32(0), LLVMInt32(element) };
LLVM_TYPE_CONST llvm::PointerType *ptrType = llvm::PointerType::get(type, 0);
llvm::Value *voidPtr = llvm::ConstantPointerNull::get(ptrType);
#if defined(LLVM_3_0) || defined(LLVM_3_0svn) || defined(LLVM_3_1svn)
llvm::ArrayRef<llvm::Value *> arrayRef(&indices[0], &indices[2]);
llvm::Instruction *gep =
llvm::GetElementPtrInst::Create(voidPtr, arrayRef, "offset_gep",
insertAtEnd);
#else
llvm::Instruction *gep =
llvm::GetElementPtrInst::Create(voidPtr, &indices[0], &indices[2],
"offset_gep", insertAtEnd);
#endif
if (is32Bit || g->opt.force32BitAddressing)
return new llvm::PtrToIntInst(gep, LLVMTypes::Int32Type,
"offset_int", insertAtEnd);
else
return new llvm::PtrToIntInst(gep, LLVMTypes::Int64Type,
"offset_int", insertAtEnd);
}
const llvm::TargetData *td = GetTargetMachine()->getTargetData();
Assert(td != NULL);
LLVM_TYPE_CONST llvm::StructType *structType =
llvm::dyn_cast<LLVM_TYPE_CONST llvm::StructType>(type);
Assert(structType != NULL);
const llvm::StructLayout *sl = td->getStructLayout(structType);
Assert(sl != NULL);
uint64_t offset = sl->getElementOffset(element);
if (is32Bit || g->opt.force32BitAddressing)
return LLVMInt32((int32_t)offset);
else
return LLVMInt64(offset);
}
/** Utility routine that defines a constant int32 with given value, adding
the symbol to both the ispc symbol table and the given LLVM module.
*/
static void
lDefineConstantInt(const char *name, int val, llvm::Module *module,
SymbolTable *symbolTable) {
Symbol *pw = new Symbol(name, SourcePos(), AtomicType::UniformConstInt32,
SC_STATIC);
pw->constValue = new ConstExpr(pw->type, val, SourcePos());
LLVM_TYPE_CONST llvm::Type *ltype = LLVMTypes::Int32Type;
llvm::Constant *linit = LLVMInt32(val);
pw->storagePtr = new llvm::GlobalVariable(*module, ltype, true,
llvm::GlobalValue::InternalLinkage,
linit, pw->name.c_str());
symbolTable->AddVariable(pw);
}
static void
lDefineConstantIntFunc(const char *name, int val, llvm::Module *module,
SymbolTable *symbolTable) {
llvm::SmallVector<const Type *, 8> args;
FunctionType *ft = new FunctionType(AtomicType::UniformInt32, args, SourcePos());
Symbol *sym = new Symbol(name, SourcePos(), ft, SC_STATIC);
llvm::Function *func = module->getFunction(name);
Assert(func != NULL); // it should be declared already...
func->addFnAttr(llvm::Attribute::AlwaysInline);
llvm::BasicBlock *bblock = llvm::BasicBlock::Create(*g->ctx, "entry", func, 0);
llvm::ReturnInst::Create(*g->ctx, LLVMInt32(val), bblock);
sym->function = func;
symbolTable->AddVariable(sym);
}
llvm::Value *
Target::StructOffset(LLVM_TYPE_CONST llvm::Type *type, int element) {
const llvm::TargetData *td = GetTargetMachine()->getTargetData();
Assert(td != NULL);
LLVM_TYPE_CONST llvm::StructType *structType =
llvm::dyn_cast<LLVM_TYPE_CONST llvm::StructType>(type);
Assert(structType != NULL);
const llvm::StructLayout *sl = td->getStructLayout(structType);
Assert(sl != NULL);
uint64_t offset = sl->getElementOffset(element);
if (is32Bit || g->opt.force32BitAddressing)
return LLVMInt32((int32_t)offset);
else
return LLVMInt64(offset);
}
/** Given the statements implementing a function, emit the code that
implements the function. Most of the work do be done here just
involves wiring up the function parameter values to be available in the
function body code.
*/
void
Function::emitCode(FunctionEmitContext *ctx, llvm::Function *function,
SourcePos firstStmtPos) {
// Connect the __mask builtin to the location in memory that stores its
// value
maskSymbol->storagePtr = ctx->GetFullMaskPointer();
// add debugging info for __mask
maskSymbol->pos = firstStmtPos;
ctx->EmitVariableDebugInfo(maskSymbol);
#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_7 // LLVM 3.7+
if (g->NoOmitFramePointer)
function->addFnAttr("no-frame-pointer-elim", "true");
#endif
#if 0
llvm::BasicBlock *entryBBlock = ctx->GetCurrentBasicBlock();
#endif
const FunctionType *type = CastType<FunctionType>(sym->type);
Assert(type != NULL);
if (type->isTask == true
#ifdef ISPC_NVPTX_ENABLED
&& (g->target->getISA() != Target::NVPTX)
#endif
){
// For tasks, there should always be three parameters: the
// pointer to the structure that holds all of the arguments, the
// thread index, and the thread count variables.
llvm::Function::arg_iterator argIter = function->arg_begin();
#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
llvm::Value *structParamPtr = argIter++;
llvm::Value *threadIndex = argIter++;
llvm::Value *threadCount = argIter++;
llvm::Value *taskIndex = argIter++;
llvm::Value *taskCount = argIter++;
llvm::Value *taskIndex0 = argIter++;
llvm::Value *taskIndex1 = argIter++;
llvm::Value *taskIndex2 = argIter++;
llvm::Value *taskCount0 = argIter++;
llvm::Value *taskCount1 = argIter++;
llvm::Value *taskCount2 = argIter++;
#else /* LLVM 3.8+ */
llvm::Value *structParamPtr = &*(argIter++);
llvm::Value *threadIndex = &*(argIter++);
llvm::Value *threadCount = &*(argIter++);
llvm::Value *taskIndex = &*(argIter++);
llvm::Value *taskCount = &*(argIter++);
llvm::Value *taskIndex0 = &*(argIter++);
llvm::Value *taskIndex1 = &*(argIter++);
llvm::Value *taskIndex2 = &*(argIter++);
llvm::Value *taskCount0 = &*(argIter++);
llvm::Value *taskCount1 = &*(argIter++);
llvm::Value *taskCount2 = &*(argIter++);
#endif
// Copy the function parameter values from the structure into local
// storage
for (unsigned int i = 0; i < args.size(); ++i)
lCopyInTaskParameter(i, structParamPtr, args, ctx);
if (type->isUnmasked == false) {
// Copy in the mask as well.
int nArgs = (int)args.size();
// The mask is the last parameter in the argument structure
llvm::Value *ptr = ctx->AddElementOffset(structParamPtr, nArgs, NULL,
"task_struct_mask");
llvm::Value *ptrval = ctx->LoadInst(ptr, "mask");
ctx->SetFunctionMask(ptrval);
}
// Copy threadIndex and threadCount into stack-allocated storage so
// that their symbols point to something reasonable.
threadIndexSym->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "threadIndex");
ctx->StoreInst(threadIndex, threadIndexSym->storagePtr);
threadCountSym->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "threadCount");
ctx->StoreInst(threadCount, threadCountSym->storagePtr);
// Copy taskIndex and taskCount into stack-allocated storage so
// that their symbols point to something reasonable.
taskIndexSym->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskIndex");
ctx->StoreInst(taskIndex, taskIndexSym->storagePtr);
taskCountSym->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskCount");
ctx->StoreInst(taskCount, taskCountSym->storagePtr);
taskIndexSym0->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskIndex0");
ctx->StoreInst(taskIndex0, taskIndexSym0->storagePtr);
taskIndexSym1->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskIndex1");
ctx->StoreInst(taskIndex1, taskIndexSym1->storagePtr);
taskIndexSym2->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskIndex2");
ctx->StoreInst(taskIndex2, taskIndexSym2->storagePtr);
taskCountSym0->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskCount0");
ctx->StoreInst(taskCount0, taskCountSym0->storagePtr);
taskCountSym1->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskCount1");
ctx->StoreInst(taskCount1, taskCountSym1->storagePtr);
taskCountSym2->storagePtr = ctx->AllocaInst(LLVMTypes::Int32Type, "taskCount2");
ctx->StoreInst(taskCount2, taskCountSym2->storagePtr);
}
else {
// Regular, non-task function
llvm::Function::arg_iterator argIter = function->arg_begin();
for (unsigned int i = 0; i < args.size(); ++i, ++argIter) {
Symbol *sym = args[i];
if (sym == NULL)
// anonymous function parameter
continue;
argIter->setName(sym->name.c_str());
// Allocate stack storage for the parameter and emit code
// to store the its value there.
sym->storagePtr = ctx->AllocaInst(argIter->getType(), sym->name.c_str());
#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
ctx->StoreInst(argIter, sym->storagePtr);
#else /* LLVM 3.8+ */
ctx->StoreInst(&*argIter, sym->storagePtr);
#endif
ctx->EmitFunctionParameterDebugInfo(sym, i);
}
// If the number of actual function arguments is equal to the
// number of declared arguments in decl->functionParams, then we
// don't have a mask parameter, so set it to be all on. This
// happens for exmaple with 'export'ed functions that the app
// calls.
if (argIter == function->arg_end()) {
Assert(type->isUnmasked || type->isExported);
ctx->SetFunctionMask(LLVMMaskAllOn);
}
else {
Assert(type->isUnmasked == false);
// Otherwise use the mask to set the entry mask value
argIter->setName("__mask");
Assert(argIter->getType() == LLVMTypes::MaskType);
#if ISPC_LLVM_VERSION <= ISPC_LLVM_3_7 /* 3.2, 3.3, 3.4, 3.5, 3.6, 3.7 */
ctx->SetFunctionMask(argIter);
#else /* LLVM 3.8+ */
ctx->SetFunctionMask(&*argIter);
#endif
Assert(++argIter == function->arg_end());
}
#ifdef ISPC_NVPTX_ENABLED
if (type->isTask == true && g->target->getISA() == Target::NVPTX)
{
llvm::NamedMDNode* annotations =
m->module->getOrInsertNamedMetadata("nvvm.annotations");
#if ISPC_LLVM_VERSION >= ISPC_LLVM_3_6 // LLVM 3.6+
llvm::SmallVector<llvm::Metadata*, 3> av;
av.push_back(llvm::ValueAsMetadata::get(function));
av.push_back(llvm::MDString::get(*g->ctx, "kernel"));
av.push_back(llvm::ConstantAsMetadata::get(LLVMInt32(1)));
annotations->addOperand(llvm::MDNode::get(*g->ctx, llvm::ArrayRef<llvm::Metadata*>(av)));
#else
llvm::SmallVector<llvm::Value*, 3> av;
av.push_back(function);
av.push_back(llvm::MDString::get(*g->ctx, "kernel"));
av.push_back(LLVMInt32(1));
annotations->addOperand(llvm::MDNode::get(*g->ctx, av));
#endif
}
#endif /* ISPC_NVPTX_ENABLED */
}
// Finally, we can generate code for the function
if (code != NULL) {
ctx->SetDebugPos(code->pos);
ctx->AddInstrumentationPoint("function entry");
int costEstimate = EstimateCost(code);
Debug(code->pos, "Estimated cost for function \"%s\" = %d\n",
sym->name.c_str(), costEstimate);
// If the body of the function is non-trivial, then we wrap the
// entire thing inside code that tests to see if the mask is all
// on, all off, or mixed. If this is a simple function, then this
// isn't worth the code bloat / overhead.
bool checkMask = (type->isTask == true) ||
(
#if ISPC_LLVM_VERSION == ISPC_LLVM_3_2 // 3.2
(function->getFnAttributes().hasAttribute(llvm::Attributes::AlwaysInline) == false)
#else // LLVM 3.3+
(function->getAttributes().getFnAttributes().hasAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::AlwaysInline) == false)
#endif
&&
costEstimate > CHECK_MASK_AT_FUNCTION_START_COST);
checkMask &= (type->isUnmasked == false);
checkMask &= (g->target->getMaskingIsFree() == false);
checkMask &= (g->opt.disableCoherentControlFlow == false);
if (checkMask) {
llvm::Value *mask = ctx->GetFunctionMask();
llvm::Value *allOn = ctx->All(mask);
llvm::BasicBlock *bbAllOn = ctx->CreateBasicBlock("all_on");
llvm::BasicBlock *bbSomeOn = ctx->CreateBasicBlock("some_on");
// Set up basic blocks for goto targets
ctx->InitializeLabelMap(code);
ctx->BranchInst(bbAllOn, bbSomeOn, allOn);
// all on: we've determined dynamically that the mask is all
// on. Set the current mask to "all on" explicitly so that
// codegen for this path can be improved with this knowledge in
// hand...
ctx->SetCurrentBasicBlock(bbAllOn);
if (!g->opt.disableMaskAllOnOptimizations)
ctx->SetFunctionMask(LLVMMaskAllOn);
code->EmitCode(ctx);
if (ctx->GetCurrentBasicBlock())
ctx->ReturnInst();
// not all on: however, at least one lane must be running,
// since we should never run with all off... some on: reset
// the mask to the value it had at function entry and emit the
// code. Resetting the mask here is important, due to the "all
// on" setting of it for the path above.
ctx->SetCurrentBasicBlock(bbSomeOn);
ctx->SetFunctionMask(mask);
// Set up basic blocks for goto targets again; we want to have
// one set of them for gotos in the 'all on' case, and a
// distinct set for the 'mixed mask' case.
ctx->InitializeLabelMap(code);
code->EmitCode(ctx);
if (ctx->GetCurrentBasicBlock())
ctx->ReturnInst();
}
else {
// Set up basic blocks for goto targets
ctx->InitializeLabelMap(code);
// No check, just emit the code
code->EmitCode(ctx);
}
}
if (ctx->GetCurrentBasicBlock()) {
// FIXME: We'd like to issue a warning if we've reached the end of
// the function without a return statement (for non-void
// functions). But the test below isn't right, since we can have
// (with 'x' a varying test) "if (x) return a; else return b;", in
// which case we have a valid basic block but its unreachable so ok
// to not have return statement.
#if 0
// If the bblock has no predecessors, then it doesn't matter if it
// doesn't have a return; it'll never be reached. If it does,
// issue a warning. Also need to warn if it's the entry block for
// the function (in which case it will not have predeccesors but is
// still reachable.)
if (type->GetReturnType()->IsVoidType() == false &&
(pred_begin(ec.bblock) != pred_end(ec.bblock) || (ec.bblock == entryBBlock)))
Warning(sym->pos, "Missing return statement in function returning \"%s\".",
type->rType->GetString().c_str());
#endif
// FIXME: would like to set the context's current position to
// e.g. the end of the function code
// if bblock is non-NULL, it hasn't been terminated by e.g. a
// return instruction. Need to add a return instruction.
ctx->ReturnInst();
}
}