GlobalVariable*
materializeStringLiteral(llvm::Module& m, const char* data)
{
Constant* ary = llvm::ConstantDataArray::getString(m.getContext(), data, true);
GlobalVariable* gv = new GlobalVariable(m, ary->getType(), true, GlobalValue::LinkageTypes::PrivateLinkage, ary, "");
gv->setConstant(true);
return gv;
}
//
// Method: runOnModule()
//
// Description:
// Entry point for this LLVM pass.
//
// Return value:
// true - The module was modified.
// false - The module was not modified.
//
bool
BreakConstantStrings::runOnModule (Module & M) {
bool modified = false;
const Type * Int8Type = IntegerType::getInt8Ty(getGlobalContext());
//
// Scan through all the global variables in the module. Mark a variable as
// non-constant if:
// o) The variable is constant
// o) The variable is an array of characters (Int8Ty).
// o) The variable is not in a special section (e.g. debug info section).
// This ensures that we don't mess up debug information or other special
// strings within the code.
//
Module::global_iterator i,e;
for (i = M.global_begin(), e = M.global_end(); i != e; ++i) {
GlobalVariable * GV = i;
//
// All global variables are pointer types. Find the type of what the
// global variable pointer is pointing at.
//
if (GV->isConstant() && (!GV->hasSection())) {
const PointerType * PT = dyn_cast<PointerType>(GV->getType());
if (const ArrayType * AT = dyn_cast<ArrayType>(PT->getElementType())) {
if (AT->getElementType() == Int8Type) {
modified = true;
++GVChanges;
GV->setConstant (false);
}
}
}
}
return modified;
}
void CodeGen_GPU_Host<CodeGen_CPU>::compile(Stmt stmt, string name,
const vector<Argument> &args,
const vector<Buffer> &images_to_embed) {
init_module();
// also set up the child codegenerator - this is set up once per
// PTX_Host::compile, and reused across multiple PTX_Dev::compile
// invocations for different kernels.
cgdev->init_module();
module = get_initial_module_for_target(target, context);
// grab runtime helper functions
// Fix the target triple
debug(1) << "Target triple of initial module: " << module->getTargetTriple() << "\n";
llvm::Triple triple = CodeGen_CPU::get_target_triple();
module->setTargetTriple(triple.str());
debug(1) << "Target triple of initial module: " << module->getTargetTriple() << "\n";
// Pass to the generic codegen
CodeGen::compile(stmt, name, args, images_to_embed);
// Unset constant flag for embedded image global variables
for (size_t i = 0; i < images_to_embed.size(); i++) {
string name = images_to_embed[i].name();
GlobalVariable *global = module->getNamedGlobal(name + ".buffer");
global->setConstant(false);
}
std::vector<char> kernel_src = cgdev->compile_to_src();
Value *kernel_src_ptr = CodeGen_CPU::create_constant_binary_blob(kernel_src, "halide_kernel_src");
// Remember the entry block so we can branch to it upon init success.
BasicBlock *entry = &function->getEntryBlock();
// Insert a new block to run initialization at the beginning of the function.
BasicBlock *init_kernels_bb = BasicBlock::Create(*context, "init_kernels",
function, entry);
builder->SetInsertPoint(init_kernels_bb);
Value *user_context = get_user_context();
Value *kernel_size = ConstantInt::get(i32, kernel_src.size());
Value *init = module->getFunction("halide_init_kernels");
internal_assert(init) << "Could not find function halide_init_kernels in initial module\n";
Value *result = builder->CreateCall4(init, user_context,
get_module_state(),
kernel_src_ptr, kernel_size);
Value *did_succeed = builder->CreateICmpEQ(result, ConstantInt::get(i32, 0));
CodeGen_CPU::create_assertion(did_succeed, "Failure inside halide_init_kernels");
// Upon success, jump to the original entry.
builder->CreateBr(entry);
// Optimize the module
CodeGen::optimize_module();
}
Module *llvm::CloneModule(const Module *M,
DenseMap<const Value*, Value*> &ValueMap) {
// First off, we need to create the new module...
Module *New = new Module(M->getModuleIdentifier());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Copy all of the type symbol table entries over.
const TypeSymbolTable &TST = M->getTypeSymbolTable();
for (TypeSymbolTable::const_iterator TI = TST.begin(), TE = TST.end();
TI != TE; ++TI)
New->addTypeName(TI->first, TI->second);
// Copy all of the dependent libraries over.
for (Module::lib_iterator I = M->lib_begin(), E = M->lib_end(); I != E; ++I)
New->addLibrary(*I);
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the ValueMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(I->getType()->getElementType(),
false,
GlobalValue::ExternalLinkage, 0,
I->getName(), New);
GV->setAlignment(I->getAlignment());
ValueMap[I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getType()->getElementType()),
GlobalValue::ExternalLinkage, I->getName(), New);
NF->copyAttributesFrom(I);
ValueMap[I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I)
ValueMap[I] = new GlobalAlias(I->getType(), GlobalAlias::ExternalLinkage,
I->getName(), NULL, New);
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(ValueMap[I]);
if (I->hasInitializer())
GV->setInitializer(cast<Constant>(MapValue(I->getInitializer(),
ValueMap)));
GV->setLinkage(I->getLinkage());
GV->setThreadLocal(I->isThreadLocal());
GV->setConstant(I->isConstant());
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(ValueMap[I]);
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
ValueMap[J] = DestI++;
}
std::vector<ReturnInst*> Returns; // Ignore returns cloned...
CloneFunctionInto(F, I, ValueMap, Returns);
}
F->setLinkage(I->getLinkage());
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
GlobalAlias *GA = cast<GlobalAlias>(ValueMap[I]);
GA->setLinkage(I->getLinkage());
if (const Constant* C = I->getAliasee())
GA->setAliasee(cast<Constant>(MapValue(C, ValueMap)));
}
return New;
}
Module *llvm::CloneModule(const Module *M,
ValueToValueMapTy &VMap) {
// First off, we need to create the new module...
Module *New = new Module(M->getModuleIdentifier(), M->getContext());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Copy all of the type symbol table entries over.
const TypeSymbolTable &TST = M->getTypeSymbolTable();
for (TypeSymbolTable::const_iterator TI = TST.begin(), TE = TST.end();
TI != TE; ++TI)
New->addTypeName(TI->first, TI->second);
// Copy all of the dependent libraries over.
for (Module::lib_iterator I = M->lib_begin(), E = M->lib_end(); I != E; ++I)
New->addLibrary(*I);
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the VMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(*New,
I->getType()->getElementType(),
false,
GlobalValue::ExternalLinkage, 0,
I->getName());
GV->setAlignment(I->getAlignment());
VMap[I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getType()->getElementType()),
GlobalValue::ExternalLinkage, I->getName(), New);
NF->copyAttributesFrom(I);
VMap[I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I)
VMap[I] = new GlobalAlias(I->getType(), GlobalAlias::ExternalLinkage,
I->getName(), NULL, New);
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(VMap[I]);
if (I->hasInitializer())
GV->setInitializer(cast<Constant>(MapValue(I->getInitializer(),
VMap)));
GV->setLinkage(I->getLinkage());
GV->setThreadLocal(I->isThreadLocal());
GV->setConstant(I->isConstant());
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(VMap[I]);
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
VMap[J] = DestI++;
}
SmallVector<ReturnInst*, 8> Returns; // Ignore returns cloned.
CloneFunctionInto(F, I, VMap, Returns);
}
F->setLinkage(I->getLinkage());
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
GlobalAlias *GA = cast<GlobalAlias>(VMap[I]);
GA->setLinkage(I->getLinkage());
if (const Constant* C = I->getAliasee())
GA->setAliasee(cast<Constant>(MapValue(C, VMap)));
}
// And named metadata....
for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
E = M->named_metadata_end(); I != E; ++I) {
const NamedMDNode &NMD = *I;
SmallVector<MDNode*, 4> MDs;
for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
MDs.push_back(cast<MDNode>(MapValue(NMD.getOperand(i), VMap)));
NamedMDNode::Create(New->getContext(), NMD.getName(),
MDs.data(), MDs.size(), New);
}
// Update metadata attach with instructions.
for (Module::iterator MI = New->begin(), ME = New->end(); MI != ME; ++MI)
for (Function::iterator FI = MI->begin(), FE = MI->end();
FI != FE; ++FI)
for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
BI != BE; ++BI) {
SmallVector<std::pair<unsigned, MDNode *>, 4 > MDs;
BI->getAllMetadata(MDs);
for (SmallVector<std::pair<unsigned, MDNode *>, 4>::iterator
MDI = MDs.begin(), MDE = MDs.end(); MDI != MDE; ++MDI) {
Value *MappedValue = MapValue(MDI->second, VMap);
if (MDI->second != MappedValue && MappedValue)
BI->setMetadata(MDI->first, cast<MDNode>(MappedValue));
}
}
return New;
}
// make_decl_llvm - Create the DECL_RTL for a VAR_DECL or FUNCTION_DECL. DECL
// should have static storage duration. In other words, it should not be an
// automatic variable, including PARM_DECLs.
//
// There is, however, one exception: this function handles variables explicitly
// placed in a particular register by the user.
//
// This function corresponds to make_decl_rtl in varasm.c, and is implicitly
// called by DECL_LLVM if a decl doesn't have an LLVM set.
//
void make_decl_llvm(tree decl) {
#ifdef ENABLE_CHECKING
// Check that we are not being given an automatic variable.
// A weak alias has TREE_PUBLIC set but not the other bits.
if (TREE_CODE(decl) == PARM_DECL || TREE_CODE(decl) == RESULT_DECL
|| (TREE_CODE(decl) == VAR_DECL && !TREE_STATIC(decl) &&
!TREE_PUBLIC(decl) && !DECL_EXTERNAL(decl) && !DECL_REGISTER(decl)))
abort();
// And that we were not given a type or a label. */
else if (TREE_CODE(decl) == TYPE_DECL || TREE_CODE(decl) == LABEL_DECL)
abort ();
#endif
// For a duplicate declaration, we can be called twice on the
// same DECL node. Don't discard the LLVM already made.
if (DECL_LLVM_SET_P(decl)) return;
if (errorcount || sorrycount)
return; // Do not process broken code.
// Global register variable with asm name, e.g.:
// register unsigned long esp __asm__("ebp");
if (TREE_CODE(decl) != FUNCTION_DECL && DECL_REGISTER(decl)) {
// This just verifies that the variable is ok. The actual "load/store"
// code paths handle accesses to the variable.
ValidateRegisterVariable(decl);
return;
}
timevar_push(TV_LLVM_GLOBALS);
const char *Name = "";
if (DECL_NAME(decl))
if (tree AssemblerName = DECL_ASSEMBLER_NAME(decl))
Name = IDENTIFIER_POINTER(AssemblerName);
// Now handle ordinary static variables and functions (in memory).
// Also handle vars declared register invalidly.
if (Name[0] == 1) {
#ifdef REGISTER_PREFIX
if (strlen (REGISTER_PREFIX) != 0) {
int reg_number = decode_reg_name(Name);
if (reg_number >= 0 || reg_number == -3)
error("%Jregister name given for non-register variable %qD",
decl, decl);
}
#endif
}
// Specifying a section attribute on a variable forces it into a
// non-.bss section, and thus it cannot be common.
if (TREE_CODE(decl) == VAR_DECL && DECL_SECTION_NAME(decl) != NULL_TREE &&
DECL_INITIAL(decl) == NULL_TREE && DECL_COMMON(decl))
DECL_COMMON(decl) = 0;
// Variables can't be both common and weak.
if (TREE_CODE(decl) == VAR_DECL && DECL_WEAK(decl))
DECL_COMMON(decl) = 0;
// Okay, now we need to create an LLVM global variable or function for this
// object. Note that this is quite possibly a forward reference to the
// object, so its type may change later.
if (TREE_CODE(decl) == FUNCTION_DECL) {
assert(Name[0] && "Function with empty name!");
// If this function has already been created, reuse the decl. This happens
// when we have something like __builtin_memset and memset in the same file.
Function *FnEntry = TheModule->getFunction(Name);
if (FnEntry == 0) {
unsigned CC;
const FunctionType *Ty =
TheTypeConverter->ConvertFunctionType(TREE_TYPE(decl), decl, NULL, CC);
FnEntry = new Function(Ty, Function::ExternalLinkage, Name, TheModule);
FnEntry->setCallingConv(CC);
// Check for external weak linkage
if (DECL_EXTERNAL(decl) && DECL_WEAK(decl))
FnEntry->setLinkage(Function::ExternalWeakLinkage);
#ifdef TARGET_ADJUST_LLVM_LINKAGE
TARGET_ADJUST_LLVM_LINKAGE(FnEntry,decl);
#endif /* TARGET_ADJUST_LLVM_LINKAGE */
// Handle visibility style
if (TREE_PUBLIC(decl)) {
if (DECL_VISIBILITY(decl) == VISIBILITY_HIDDEN)
FnEntry->setVisibility(GlobalValue::HiddenVisibility);
else if (DECL_VISIBILITY(decl) == VISIBILITY_PROTECTED)
FnEntry->setVisibility(GlobalValue::ProtectedVisibility);
}
assert(FnEntry->getName() == Name &&"Preexisting fn with the same name!");
}
SET_DECL_LLVM(decl, FnEntry);
} else {
assert((TREE_CODE(decl) == VAR_DECL ||
TREE_CODE(decl) == CONST_DECL) && "Not a function or var decl?");
const Type *Ty = ConvertType(TREE_TYPE(decl));
GlobalVariable *GV ;
// If we have "extern void foo", make the global have type {} instead of
// type void.
if (Ty == Type::VoidTy)
Ty = StructType::get(std::vector<const Type*>(), false);
if (Name[0] == 0) { // Global has no name.
GV = new GlobalVariable(Ty, false, GlobalValue::ExternalLinkage, 0,
"", TheModule);
// Check for external weak linkage
if (DECL_EXTERNAL(decl) && DECL_WEAK(decl))
GV->setLinkage(GlobalValue::ExternalWeakLinkage);
#ifdef TARGET_ADJUST_LLVM_LINKAGE
TARGET_ADJUST_LLVM_LINKAGE(GV,decl);
#endif /* TARGET_ADJUST_LLVM_LINKAGE */
// Handle visibility style
if (TREE_PUBLIC(decl)) {
if (DECL_VISIBILITY(decl) == VISIBILITY_HIDDEN)
GV->setVisibility(GlobalValue::HiddenVisibility);
else if (DECL_VISIBILITY(decl) == VISIBILITY_PROTECTED)
GV->setVisibility(GlobalValue::ProtectedVisibility);
}
} else {
// If the global has a name, prevent multiple vars with the same name from
// being created.
GlobalVariable *GVE = TheModule->getGlobalVariable(Name);
if (GVE == 0) {
GV = new GlobalVariable(Ty, false, GlobalValue::ExternalLinkage,0,
Name, TheModule);
// Check for external weak linkage
if (DECL_EXTERNAL(decl) && DECL_WEAK(decl))
GV->setLinkage(GlobalValue::ExternalWeakLinkage);
#ifdef TARGET_ADJUST_LLVM_LINKAGE
TARGET_ADJUST_LLVM_LINKAGE(GV,decl);
#endif /* TARGET_ADJUST_LLVM_LINKAGE */
// Handle visibility style
if (TREE_PUBLIC(decl)) {
if (DECL_VISIBILITY(decl) == VISIBILITY_HIDDEN)
GV->setVisibility(GlobalValue::HiddenVisibility);
else if (DECL_VISIBILITY(decl) == VISIBILITY_PROTECTED)
GV->setVisibility(GlobalValue::ProtectedVisibility);
}
// If GV got renamed, then there is already an object with this name in
// the symbol table. If this happens, the old one must be a forward
// decl, just replace it with a cast of the new one.
if (GV->getName() != Name) {
Function *F = TheModule->getFunction(Name);
assert(F && F->isDeclaration() && "A function turned into a global?");
// Replace any uses of "F" with uses of GV.
Value *FInNewType = ConstantExpr::getBitCast(GV, F->getType());
F->replaceAllUsesWith(FInNewType);
// Update the decl that points to F.
changeLLVMValue(F, FInNewType);
// Now we can give GV the proper name.
GV->takeName(F);
// F is now dead, nuke it.
F->eraseFromParent();
}
} else {
GV = GVE; // Global already created, reuse it.
}
}
if ((TREE_READONLY(decl) && !TREE_SIDE_EFFECTS(decl)) ||
TREE_CODE(decl) == CONST_DECL) {
if (DECL_EXTERNAL(decl)) {
// Mark external globals constant even though they could be marked
// non-constant in the defining translation unit. The definition of the
// global determines whether the global is ultimately constant or not,
// marking this constant will allow us to do some extra (legal)
// optimizations that we would otherwise not be able to do. (In C++,
// any global that is 'C++ const' may not be readonly: it could have a
// dynamic initializer.
//
GV->setConstant(true);
} else {
// Mark readonly globals with constant initializers constant.
if (DECL_INITIAL(decl) != error_mark_node && // uninitialized?
DECL_INITIAL(decl) &&
(TREE_CONSTANT(DECL_INITIAL(decl)) ||
TREE_CODE(DECL_INITIAL(decl)) == STRING_CST))
GV->setConstant(true);
}
}
// Set thread local (TLS)
if (TREE_CODE(decl) == VAR_DECL && DECL_THREAD_LOCAL(decl))
GV->setThreadLocal(true);
SET_DECL_LLVM(decl, GV);
}
timevar_pop(TV_LLVM_GLOBALS);
}
bool LowerEmuTLS::addEmuTlsVar(Module &M, const GlobalVariable *GV) {
LLVMContext &C = M.getContext();
PointerType *VoidPtrType = Type::getInt8PtrTy(C);
std::string EmuTlsVarName = ("__emutls_v." + GV->getName()).str();
GlobalVariable *EmuTlsVar = M.getNamedGlobal(EmuTlsVarName);
if (EmuTlsVar)
return false; // It has been added before.
const DataLayout &DL = M.getDataLayout();
Constant *NullPtr = ConstantPointerNull::get(VoidPtrType);
// Get non-zero initializer from GV's initializer.
const Constant *InitValue = nullptr;
if (GV->hasInitializer()) {
InitValue = GV->getInitializer();
const ConstantInt *InitIntValue = dyn_cast<ConstantInt>(InitValue);
// When GV's init value is all 0, omit the EmuTlsTmplVar and let
// the emutls library function to reset newly allocated TLS variables.
if (isa<ConstantAggregateZero>(InitValue) ||
(InitIntValue && InitIntValue->isZero()))
InitValue = nullptr;
}
// Create the __emutls_v. symbol, whose type has 4 fields:
// word size; // size of GV in bytes
// word align; // alignment of GV
// void *ptr; // initialized to 0; set at run time per thread.
// void *templ; // 0 or point to __emutls_t.*
// sizeof(word) should be the same as sizeof(void*) on target.
IntegerType *WordType = DL.getIntPtrType(C);
PointerType *InitPtrType = InitValue ?
PointerType::getUnqual(InitValue->getType()) : VoidPtrType;
Type *ElementTypes[4] = {WordType, WordType, VoidPtrType, InitPtrType};
ArrayRef<Type*> ElementTypeArray(ElementTypes, 4);
StructType *EmuTlsVarType = StructType::create(ElementTypeArray);
EmuTlsVar = cast<GlobalVariable>(
M.getOrInsertGlobal(EmuTlsVarName, EmuTlsVarType));
copyLinkageVisibility(M, GV, EmuTlsVar);
// Define "__emutls_t.*" and "__emutls_v.*" only if GV is defined.
if (!GV->hasInitializer())
return true;
Type *GVType = GV->getValueType();
unsigned GVAlignment = GV->getAlignment();
if (!GVAlignment) {
// When LLVM IL declares a variable without alignment, use
// the ABI default alignment for the type.
GVAlignment = DL.getABITypeAlignment(GVType);
}
// Define "__emutls_t.*" if there is InitValue
GlobalVariable *EmuTlsTmplVar = nullptr;
if (InitValue) {
std::string EmuTlsTmplName = ("__emutls_t." + GV->getName()).str();
EmuTlsTmplVar = dyn_cast_or_null<GlobalVariable>(
M.getOrInsertGlobal(EmuTlsTmplName, GVType));
assert(EmuTlsTmplVar && "Failed to create emualted TLS initializer");
EmuTlsTmplVar->setConstant(true);
EmuTlsTmplVar->setInitializer(const_cast<Constant*>(InitValue));
EmuTlsTmplVar->setAlignment(GVAlignment);
copyLinkageVisibility(M, GV, EmuTlsTmplVar);
}
// Define "__emutls_v.*" with initializer and alignment.
Constant *ElementValues[4] = {
ConstantInt::get(WordType, DL.getTypeStoreSize(GVType)),
ConstantInt::get(WordType, GVAlignment),
NullPtr, EmuTlsTmplVar ? EmuTlsTmplVar : NullPtr
};
ArrayRef<Constant*> ElementValueArray(ElementValues, 4);
EmuTlsVar->setInitializer(
ConstantStruct::get(EmuTlsVarType, ElementValueArray));
unsigned MaxAlignment = std::max(
DL.getABITypeAlignment(WordType),
DL.getABITypeAlignment(VoidPtrType));
EmuTlsVar->setAlignment(MaxAlignment);
return true;
}