void LTOCodeGenerator::applyScopeRestrictions() {
if (_scopeRestrictionsDone) return;
Module *mergedModule = _linker.getModule();
// Start off with a verification pass.
PassManager passes;
passes.add(createVerifierPass());
// mark which symbols can not be internalized
MCContext Context(*_target->getMCAsmInfo(), *_target->getRegisterInfo(),NULL);
Mangler mangler(Context, *_target->getDataLayout());
std::vector<const char*> mustPreserveList;
SmallPtrSet<GlobalValue*, 8> asmUsed;
for (Module::iterator f = mergedModule->begin(),
e = mergedModule->end(); f != e; ++f)
applyRestriction(*f, mustPreserveList, asmUsed, mangler);
for (Module::global_iterator v = mergedModule->global_begin(),
e = mergedModule->global_end(); v != e; ++v)
applyRestriction(*v, mustPreserveList, asmUsed, mangler);
for (Module::alias_iterator a = mergedModule->alias_begin(),
e = mergedModule->alias_end(); a != e; ++a)
applyRestriction(*a, mustPreserveList, asmUsed, mangler);
GlobalVariable *LLVMCompilerUsed =
mergedModule->getGlobalVariable("llvm.compiler.used");
findUsedValues(LLVMCompilerUsed, asmUsed);
if (LLVMCompilerUsed)
LLVMCompilerUsed->eraseFromParent();
if (!asmUsed.empty()) {
llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(_context);
std::vector<Constant*> asmUsed2;
for (SmallPtrSet<GlobalValue*, 16>::const_iterator i = asmUsed.begin(),
e = asmUsed.end(); i !=e; ++i) {
GlobalValue *GV = *i;
Constant *c = ConstantExpr::getBitCast(GV, i8PTy);
asmUsed2.push_back(c);
}
llvm::ArrayType *ATy = llvm::ArrayType::get(i8PTy, asmUsed2.size());
LLVMCompilerUsed =
new llvm::GlobalVariable(*mergedModule, ATy, false,
llvm::GlobalValue::AppendingLinkage,
llvm::ConstantArray::get(ATy, asmUsed2),
"llvm.compiler.used");
LLVMCompilerUsed->setSection("llvm.metadata");
}
// Add prerequisite passes needed by SAFECode
PassManagerBuilder().populateLTOPassManager(passes, /*Internalize=*/ false,
!DisableInline);
passes.add(createInternalizePass(mustPreserveList));
// apply scope restrictions
passes.run(*mergedModule);
_scopeRestrictionsDone = true;
}
// destructively move the contents of src into dest
// this assumes that the targets of the two modules are the same
// including the DataLayout and ModuleFlags (for example)
// and that there is no module-level assembly
static void jl_merge_module(Module *dest, std::unique_ptr<Module> src)
{
assert(dest != src.get());
for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) {
GlobalVariable *sG = &*I;
GlobalValue *dG = dest->getNamedValue(sG->getName());
++I;
if (dG) {
if (sG->isDeclaration()) {
sG->replaceAllUsesWith(dG);
sG->eraseFromParent();
continue;
}
else {
dG->replaceAllUsesWith(sG);
dG->eraseFromParent();
}
}
sG->removeFromParent();
dest->getGlobalList().push_back(sG);
}
for (Module::iterator I = src->begin(), E = src->end(); I != E;) {
Function *sG = &*I;
GlobalValue *dG = dest->getNamedValue(sG->getName());
++I;
if (dG) {
if (sG->isDeclaration()) {
sG->replaceAllUsesWith(dG);
sG->eraseFromParent();
continue;
}
else {
dG->replaceAllUsesWith(sG);
dG->eraseFromParent();
}
}
sG->removeFromParent();
dest->getFunctionList().push_back(sG);
}
for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) {
GlobalAlias *sG = &*I;
GlobalValue *dG = dest->getNamedValue(sG->getName());
++I;
if (dG) {
if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two
sG->replaceAllUsesWith(dG);
sG->eraseFromParent();
continue;
}
else {
dG->replaceAllUsesWith(sG);
dG->eraseFromParent();
}
}
sG->removeFromParent();
dest->getAliasList().push_back(sG);
}
// metadata nodes need to be explicitly merged not just copied
// so there are special passes here for each known type of metadata
NamedMDNode *sNMD = src->getNamedMetadata("llvm.dbg.cu");
if (sNMD) {
NamedMDNode *dNMD = dest->getOrInsertNamedMetadata("llvm.dbg.cu");
#ifdef LLVM35
for (NamedMDNode::op_iterator I = sNMD->op_begin(), E = sNMD->op_end(); I != E; ++I) {
dNMD->addOperand(*I);
}
#else
for (unsigned i = 0, l = sNMD->getNumOperands(); i < l; i++) {
dNMD->addOperand(sNMD->getOperand(i));
}
#endif
}
}
bool AllocateDataSegment::runOnModule(Module &M) {
DataLayout DL(&M);
Type *I8 = Type::getInt8Ty(M.getContext());
Type *I32 = Type::getInt32Ty(M.getContext());
Type *IntPtrType = DL.getIntPtrType(M.getContext());
// First, we do a pass over the global variables, in which we compute
// the amount of required padding between them and consequently their
// addresses relative to the memory base of the sandbox. References to each
// global are then replaced with direct memory pointers.
uint32_t VarOffset = 0;
DenseMap<GlobalVariable*, uint32_t> VarPadding;
for (Module::global_iterator GV = M.global_begin(), E = M.global_end();
GV != E; ++GV) {
assert(GV->hasInitializer());
uint32_t Padding = getPadding(VarOffset, GV, DL);
VarPadding[GV] = Padding;
VarOffset += Padding;
GV->replaceAllUsesWith(
ConstantExpr::getIntToPtr(
ConstantInt::get(IntPtrType,
DataSegmentBaseAddress + VarOffset),
GV->getType()));
VarOffset += DL.getTypeStoreSize(GV->getType()->getPointerElementType());
}
// Using the offsets computed above, we prepare the layout and the contents
// of the desired data structure. After the type and initializer of each
// global is copied, the global is not needed any more and it is erased.
SmallVector<Type*, 10> TemplateLayout;
SmallVector<Constant*, 10> TemplateData;
for (Module::global_iterator It = M.global_begin(), E = M.global_end();
It != E; ) {
GlobalVariable *GV = It++;
uint32_t Padding = VarPadding[GV];
if (Padding > 0) {
Type *PaddingType = ArrayType::get(I8, Padding);
TemplateLayout.push_back(PaddingType);
TemplateData.push_back(ConstantAggregateZero::get(PaddingType));
}
TemplateLayout.push_back(GV->getType()->getPointerElementType());
TemplateData.push_back(GV->getInitializer());
GV->eraseFromParent();
}
// Finally, we create the struct and size global variables.
StructType *TemplateType =
StructType::create(M.getContext(), ExternalSymName_DataSegment);
TemplateType->setBody(TemplateLayout, /*isPacked=*/true);
Constant *Template = ConstantStruct::get(TemplateType, TemplateData);
new GlobalVariable(M, Template->getType(), /*isConstant=*/true,
GlobalVariable::ExternalLinkage, Template,
ExternalSymName_DataSegment);
Constant *TemplateSize =
ConstantInt::get(I32, DL.getTypeAllocSize(TemplateType));
new GlobalVariable(M, TemplateSize->getType(), /*isConstant=*/true,
GlobalVariable::ExternalLinkage, TemplateSize,
ExternalSymName_DataSegmentSize);
return true;
}
void InitializeSoftBound:: constructCheckHandlers(Module & module){
Type* void_ty = Type::getVoidTy(module.getContext());
Type* void_ptr_ty = PointerType::getUnqual(Type::getInt8Ty(module.getContext()));
Type* size_ty = Type::getInt64Ty(module.getContext());
module.getOrInsertFunction("__softboundcets_spatial_load_dereference_check",
void_ty, void_ptr_ty, void_ptr_ty,
void_ptr_ty, size_ty, NULL);
module.getOrInsertFunction("__softboundcets_spatial_store_dereference_check",
void_ty, void_ptr_ty, void_ptr_ty,
void_ptr_ty, size_ty, NULL);
module.getOrInsertFunction("__softboundcets_temporal_load_dereference_check",
void_ty, void_ptr_ty, size_ty,
void_ptr_ty, void_ptr_ty, NULL);
module.getOrInsertFunction("__softboundcets_temporal_store_dereference_check",
void_ty, void_ptr_ty, size_ty,
void_ptr_ty, void_ptr_ty, NULL);
Function* global_init = (Function *) module.getOrInsertFunction("__softboundcets_global_init",
void_ty, NULL);
global_init->setDoesNotThrow();
global_init->setLinkage(GlobalValue::InternalLinkage);
BasicBlock* BB = BasicBlock::Create(module.getContext(),
"entry", global_init);
Function* softboundcets_init = (Function*) module.getOrInsertFunction("__softboundcets_init", void_ty, Type::getInt32Ty(module.getContext()), NULL);
SmallVector<Value*, 8> args;
Constant * const_one = ConstantInt::get(Type::getInt32Ty(module.getContext()), 1);
args.push_back(const_one);
Instruction* ret = ReturnInst::Create(module.getContext(), BB);
CallInst::Create(softboundcets_init, args, "", ret);
Type * Int32Type = IntegerType::getInt32Ty(module.getContext());
std::vector<Constant *> CtorInits;
CtorInits.push_back(ConstantInt::get(Int32Type, 0));
CtorInits.push_back(global_init);
StructType * ST = ConstantStruct::getTypeForElements(CtorInits, false);
Constant * RuntimeCtorInit = ConstantStruct::get(ST, CtorInits);
//
// Get the current set of static global constructors and add the new ctor
// to the list.
//
std::vector<Constant *> CurrentCtors;
GlobalVariable * GVCtor = module.getNamedGlobal ("llvm.global_ctors");
if (GVCtor) {
if (Constant * C = GVCtor->getInitializer()) {
for (unsigned index = 0; index < C->getNumOperands(); ++index) {
CurrentCtors.push_back (dyn_cast<Constant>(C->getOperand (index)));
}
}
}
CurrentCtors.push_back(RuntimeCtorInit);
//
// Create a new initializer.
//
ArrayType * AT = ArrayType::get (RuntimeCtorInit-> getType(),
CurrentCtors.size());
Constant * NewInit = ConstantArray::get (AT, CurrentCtors);
//
// Create the new llvm.global_ctors global variable and remove the old one
// if it existed.
//
Value * newGVCtor = new GlobalVariable (module,
NewInit->getType(),
false,
GlobalValue::AppendingLinkage,
NewInit,
"llvm.global_ctors");
if (GVCtor) {
newGVCtor->takeName (GVCtor);
GVCtor->eraseFromParent ();
}
}
bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
Module &M, bool isConst, unsigned AddrSpace) const {
auto &DL = M.getDataLayout();
// FIXME: Find better heuristics
std::stable_sort(
Globals.begin(), Globals.end(),
[&DL](const GlobalVariable *GV1, const GlobalVariable *GV2) {
Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
return (DL.getTypeAllocSize(Ty1) < DL.getTypeAllocSize(Ty2));
});
// If we want to just blindly group all globals together, do so.
if (!GlobalMergeGroupByUse) {
BitVector AllGlobals(Globals.size());
AllGlobals.set();
return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
}
// If we want to be smarter, look at all uses of each global, to try to
// discover all sets of globals used together, and how many times each of
// these sets occured.
//
// Keep this reasonably efficient, by having an append-only list of all sets
// discovered so far (UsedGlobalSet), and mapping each "together-ness" unit of
// code (currently, a Function) to the set of globals seen so far that are
// used together in that unit (GlobalUsesByFunction).
//
// When we look at the Nth global, we now that any new set is either:
// - the singleton set {N}, containing this global only, or
// - the union of {N} and a previously-discovered set, containing some
// combination of the previous N-1 globals.
// Using that knowledge, when looking at the Nth global, we can keep:
// - a reference to the singleton set {N} (CurGVOnlySetIdx)
// - a list mapping each previous set to its union with {N} (EncounteredUGS),
// if it actually occurs.
// We keep track of the sets of globals used together "close enough".
struct UsedGlobalSet {
UsedGlobalSet(size_t Size) : Globals(Size), UsageCount(1) {}
BitVector Globals;
unsigned UsageCount;
};
// Each set is unique in UsedGlobalSets.
std::vector<UsedGlobalSet> UsedGlobalSets;
// Avoid repeating the create-global-set pattern.
auto CreateGlobalSet = [&]() -> UsedGlobalSet & {
UsedGlobalSets.emplace_back(Globals.size());
return UsedGlobalSets.back();
};
// The first set is the empty set.
CreateGlobalSet().UsageCount = 0;
// We define "close enough" to be "in the same function".
// FIXME: Grouping uses by function is way too aggressive, so we should have
// a better metric for distance between uses.
// The obvious alternative would be to group by BasicBlock, but that's in
// turn too conservative..
// Anything in between wouldn't be trivial to compute, so just stick with
// per-function grouping.
// The value type is an index into UsedGlobalSets.
// The default (0) conveniently points to the empty set.
DenseMap<Function *, size_t /*UsedGlobalSetIdx*/> GlobalUsesByFunction;
// Now, look at each merge-eligible global in turn.
// Keep track of the sets we already encountered to which we added the
// current global.
// Each element matches the same-index element in UsedGlobalSets.
// This lets us efficiently tell whether a set has already been expanded to
// include the current global.
std::vector<size_t> EncounteredUGS;
for (size_t GI = 0, GE = Globals.size(); GI != GE; ++GI) {
GlobalVariable *GV = Globals[GI];
// Reset the encountered sets for this global...
std::fill(EncounteredUGS.begin(), EncounteredUGS.end(), 0);
// ...and grow it in case we created new sets for the previous global.
EncounteredUGS.resize(UsedGlobalSets.size());
// We might need to create a set that only consists of the current global.
// Keep track of its index into UsedGlobalSets.
size_t CurGVOnlySetIdx = 0;
// For each global, look at all its Uses.
for (auto &U : GV->uses()) {
// This Use might be a ConstantExpr. We're interested in Instruction
// users, so look through ConstantExpr...
Use *UI, *UE;
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U.getUser())) {
if (CE->use_empty())
continue;
UI = &*CE->use_begin();
UE = nullptr;
} else if (isa<Instruction>(U.getUser())) {
UI = &U;
UE = UI->getNext();
} else {
continue;
}
// ...to iterate on all the instruction users of the global.
// Note that we iterate on Uses and not on Users to be able to getNext().
for (; UI != UE; UI = UI->getNext()) {
Instruction *I = dyn_cast<Instruction>(UI->getUser());
if (!I)
continue;
Function *ParentFn = I->getParent()->getParent();
// If we're only optimizing for size, ignore non-minsize functions.
if (OnlyOptimizeForSize &&
!ParentFn->hasFnAttribute(Attribute::MinSize))
continue;
size_t UGSIdx = GlobalUsesByFunction[ParentFn];
// If this is the first global the basic block uses, map it to the set
// consisting of this global only.
if (!UGSIdx) {
// If that set doesn't exist yet, create it.
if (!CurGVOnlySetIdx) {
CurGVOnlySetIdx = UsedGlobalSets.size();
CreateGlobalSet().Globals.set(GI);
} else {
++UsedGlobalSets[CurGVOnlySetIdx].UsageCount;
}
GlobalUsesByFunction[ParentFn] = CurGVOnlySetIdx;
continue;
}
// If we already encountered this BB, just increment the counter.
if (UsedGlobalSets[UGSIdx].Globals.test(GI)) {
++UsedGlobalSets[UGSIdx].UsageCount;
continue;
}
// If not, the previous set wasn't actually used in this function.
--UsedGlobalSets[UGSIdx].UsageCount;
// If we already expanded the previous set to include this global, just
// reuse that expanded set.
if (size_t ExpandedIdx = EncounteredUGS[UGSIdx]) {
++UsedGlobalSets[ExpandedIdx].UsageCount;
GlobalUsesByFunction[ParentFn] = ExpandedIdx;
continue;
}
// If not, create a new set consisting of the union of the previous set
// and this global. Mark it as encountered, so we can reuse it later.
GlobalUsesByFunction[ParentFn] = EncounteredUGS[UGSIdx] =
UsedGlobalSets.size();
UsedGlobalSet &NewUGS = CreateGlobalSet();
NewUGS.Globals.set(GI);
NewUGS.Globals |= UsedGlobalSets[UGSIdx].Globals;
}
}
}
// Now we found a bunch of sets of globals used together. We accumulated
// the number of times we encountered the sets (i.e., the number of blocks
// that use that exact set of globals).
//
// Multiply that by the size of the set to give us a crude profitability
// metric.
std::sort(UsedGlobalSets.begin(), UsedGlobalSets.end(),
[](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) {
return UGS1.Globals.count() * UGS1.UsageCount <
UGS2.Globals.count() * UGS2.UsageCount;
});
// We can choose to merge all globals together, but ignore globals never used
// with another global. This catches the obviously non-profitable cases of
// having a single global, but is aggressive enough for any other case.
if (GlobalMergeIgnoreSingleUse) {
BitVector AllGlobals(Globals.size());
for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) {
const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1];
if (UGS.UsageCount == 0)
continue;
if (UGS.Globals.count() > 1)
AllGlobals |= UGS.Globals;
}
return doMerge(Globals, AllGlobals, M, isConst, AddrSpace);
}
// Starting from the sets with the best (=biggest) profitability, find a
// good combination.
// The ideal (and expensive) solution can only be found by trying all
// combinations, looking for the one with the best profitability.
// Don't be smart about it, and just pick the first compatible combination,
// starting with the sets with the best profitability.
BitVector PickedGlobals(Globals.size());
bool Changed = false;
for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) {
const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1];
if (UGS.UsageCount == 0)
continue;
if (PickedGlobals.anyCommon(UGS.Globals))
continue;
PickedGlobals |= UGS.Globals;
// If the set only contains one global, there's no point in merging.
// Ignore the global for inclusion in other sets though, so keep it in
// PickedGlobals.
if (UGS.Globals.count() < 2)
continue;
Changed |= doMerge(Globals, UGS.Globals, M, isConst, AddrSpace);
}
return Changed;
}
// FIXME: Should try to pick the most likely to be profitable allocas first.
bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
// Array allocations are probably not worth handling, since an allocation of
// the array type is the canonical form.
if (!I.isStaticAlloca() || I.isArrayAllocation())
return false;
IRBuilder<> Builder(&I);
// First try to replace the alloca with a vector
Type *AllocaTy = I.getAllocatedType();
DEBUG(dbgs() << "Trying to promote " << I << '\n');
if (tryPromoteAllocaToVector(&I, AS))
return true; // Promoted to vector.
const Function &ContainingFunction = *I.getParent()->getParent();
CallingConv::ID CC = ContainingFunction.getCallingConv();
// Don't promote the alloca to LDS for shader calling conventions as the work
// item ID intrinsics are not supported for these calling conventions.
// Furthermore not all LDS is available for some of the stages.
switch (CC) {
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
break;
default:
DEBUG(dbgs() << " promote alloca to LDS not supported with calling convention.\n");
return false;
}
// Not likely to have sufficient local memory for promotion.
if (!SufficientLDS)
return false;
const AMDGPUSubtarget &ST =
TM->getSubtarget<AMDGPUSubtarget>(ContainingFunction);
unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(ContainingFunction).second;
const DataLayout &DL = Mod->getDataLayout();
unsigned Align = I.getAlignment();
if (Align == 0)
Align = DL.getABITypeAlignment(I.getAllocatedType());
// FIXME: This computed padding is likely wrong since it depends on inverse
// usage order.
//
// FIXME: It is also possible that if we're allowed to use all of the memory
// could could end up using more than the maximum due to alignment padding.
uint32_t NewSize = alignTo(CurrentLocalMemUsage, Align);
uint32_t AllocSize = WorkGroupSize * DL.getTypeAllocSize(AllocaTy);
NewSize += AllocSize;
if (NewSize > LocalMemLimit) {
DEBUG(dbgs() << " " << AllocSize
<< " bytes of local memory not available to promote\n");
return false;
}
CurrentLocalMemUsage = NewSize;
std::vector<Value*> WorkList;
if (!collectUsesWithPtrTypes(&I, &I, WorkList)) {
DEBUG(dbgs() << " Do not know how to convert all uses\n");
return false;
}
DEBUG(dbgs() << "Promoting alloca to local memory\n");
Function *F = I.getParent()->getParent();
Type *GVTy = ArrayType::get(I.getAllocatedType(), WorkGroupSize);
GlobalVariable *GV = new GlobalVariable(
*Mod, GVTy, false, GlobalValue::InternalLinkage,
UndefValue::get(GVTy),
Twine(F->getName()) + Twine('.') + I.getName(),
nullptr,
GlobalVariable::NotThreadLocal,
AS.LOCAL_ADDRESS);
GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
GV->setAlignment(I.getAlignment());
Value *TCntY, *TCntZ;
std::tie(TCntY, TCntZ) = getLocalSizeYZ(Builder);
Value *TIdX = getWorkitemID(Builder, 0);
Value *TIdY = getWorkitemID(Builder, 1);
Value *TIdZ = getWorkitemID(Builder, 2);
Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ, "", true, true);
Tmp0 = Builder.CreateMul(Tmp0, TIdX);
Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ, "", true, true);
Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
TID = Builder.CreateAdd(TID, TIdZ);
Value *Indices[] = {
Constant::getNullValue(Type::getInt32Ty(Mod->getContext())),
TID
};
Value *Offset = Builder.CreateInBoundsGEP(GVTy, GV, Indices);
I.mutateType(Offset->getType());
I.replaceAllUsesWith(Offset);
I.eraseFromParent();
for (Value *V : WorkList) {
CallInst *Call = dyn_cast<CallInst>(V);
if (!Call) {
if (ICmpInst *CI = dyn_cast<ICmpInst>(V)) {
Value *Src0 = CI->getOperand(0);
Type *EltTy = Src0->getType()->getPointerElementType();
PointerType *NewTy = PointerType::get(EltTy, AS.LOCAL_ADDRESS);
if (isa<ConstantPointerNull>(CI->getOperand(0)))
CI->setOperand(0, ConstantPointerNull::get(NewTy));
if (isa<ConstantPointerNull>(CI->getOperand(1)))
CI->setOperand(1, ConstantPointerNull::get(NewTy));
continue;
}
// The operand's value should be corrected on its own and we don't want to
// touch the users.
if (isa<AddrSpaceCastInst>(V))
continue;
Type *EltTy = V->getType()->getPointerElementType();
PointerType *NewTy = PointerType::get(EltTy, AS.LOCAL_ADDRESS);
// FIXME: It doesn't really make sense to try to do this for all
// instructions.
V->mutateType(NewTy);
// Adjust the types of any constant operands.
if (SelectInst *SI = dyn_cast<SelectInst>(V)) {
if (isa<ConstantPointerNull>(SI->getOperand(1)))
SI->setOperand(1, ConstantPointerNull::get(NewTy));
if (isa<ConstantPointerNull>(SI->getOperand(2)))
SI->setOperand(2, ConstantPointerNull::get(NewTy));
} else if (PHINode *Phi = dyn_cast<PHINode>(V)) {
for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
if (isa<ConstantPointerNull>(Phi->getIncomingValue(I)))
Phi->setIncomingValue(I, ConstantPointerNull::get(NewTy));
}
}
continue;
}
IntrinsicInst *Intr = cast<IntrinsicInst>(Call);
Builder.SetInsertPoint(Intr);
switch (Intr->getIntrinsicID()) {
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
// These intrinsics are for address space 0 only
Intr->eraseFromParent();
continue;
case Intrinsic::memcpy: {
MemCpyInst *MemCpy = cast<MemCpyInst>(Intr);
Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getDestAlignment(),
MemCpy->getRawSource(), MemCpy->getSourceAlignment(),
MemCpy->getLength(), MemCpy->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memmove: {
MemMoveInst *MemMove = cast<MemMoveInst>(Intr);
Builder.CreateMemMove(MemMove->getRawDest(), MemMove->getDestAlignment(),
MemMove->getRawSource(), MemMove->getSourceAlignment(),
MemMove->getLength(), MemMove->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memset: {
MemSetInst *MemSet = cast<MemSetInst>(Intr);
Builder.CreateMemSet(MemSet->getRawDest(), MemSet->getValue(),
MemSet->getLength(), MemSet->getDestAlignment(),
MemSet->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::invariant_start:
case Intrinsic::invariant_end:
case Intrinsic::invariant_group_barrier:
Intr->eraseFromParent();
// FIXME: I think the invariant marker should still theoretically apply,
// but the intrinsics need to be changed to accept pointers with any
// address space.
continue;
case Intrinsic::objectsize: {
Value *Src = Intr->getOperand(0);
Type *SrcTy = Src->getType()->getPointerElementType();
Function *ObjectSize = Intrinsic::getDeclaration(Mod,
Intrinsic::objectsize,
{ Intr->getType(), PointerType::get(SrcTy, AS.LOCAL_ADDRESS) }
);
CallInst *NewCall = Builder.CreateCall(
ObjectSize, {Src, Intr->getOperand(1), Intr->getOperand(2)});
Intr->replaceAllUsesWith(NewCall);
Intr->eraseFromParent();
continue;
}
default:
Intr->print(errs());
llvm_unreachable("Don't know how to promote alloca intrinsic use.");
}
}
return true;
}
Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
// First off, we need to create the new module.
Module *New = new Module(M->getModuleIdentifier(), M->getContext());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the VMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(*New,
I->getType()->getElementType(),
I->isConstant(), I->getLinkage(),
(Constant*) nullptr, I->getName(),
(GlobalVariable*) nullptr,
I->getThreadLocalMode(),
I->getType()->getAddressSpace());
GV->copyAttributesFrom(I);
VMap[I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getType()->getElementType()),
I->getLinkage(), I->getName(), New);
NF->copyAttributesFrom(I);
VMap[I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
auto *PTy = cast<PointerType>(I->getType());
auto *GA =
GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
I->getLinkage(), I->getName(), New);
GA->copyAttributesFrom(I);
VMap[I] = GA;
}
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(VMap[I]);
if (I->hasInitializer())
GV->setInitializer(MapValue(I->getInitializer(), VMap));
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(VMap[I]);
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
VMap[J] = DestI++;
}
SmallVector<ReturnInst*, 8> Returns; // Ignore returns cloned.
CloneFunctionInto(F, I, VMap, /*ModuleLevelChanges=*/true, Returns);
}
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
GlobalAlias *GA = cast<GlobalAlias>(VMap[I]);
if (const Constant *C = I->getAliasee())
GA->setAliasee(cast<GlobalObject>(MapValue(C, VMap)));
}
// And named metadata....
for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
E = M->named_metadata_end(); I != E; ++I) {
const NamedMDNode &NMD = *I;
NamedMDNode *NewNMD = New->getOrInsertNamedMetadata(NMD.getName());
for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
NewNMD->addOperand(MapValue(NMD.getOperand(i), VMap));
}
return New;
}
void SMT12Writer::declareGlobalVariable(const GlobalVariable & v) {
m_out << ":extrafuns ((" << NameDecorator<Value>(&v)
<< " " << NameDecorator<Type>(v.getType()) << "))\n";
}
int main(int argc, char **argv) {
// Print a stack trace if we signal out.
sys::PrintStackTraceOnErrorSignal();
PrettyStackTraceProgram X(argc, argv);
LLVMContext &Context = getGlobalContext();
llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
cl::ParseCommandLineOptions(argc, argv, "llvm extractor\n");
// Use lazy loading, since we only care about selected global values.
SMDiagnostic Err;
std::auto_ptr<Module> M;
M.reset(getLazyIRFileModule(InputFilename, Err, Context));
if (M.get() == 0) {
Err.print(argv[0], errs());
return 1;
}
// Use SetVector to avoid duplicates.
SetVector<GlobalValue *> GVs;
// Figure out which globals we should extract.
for (size_t i = 0, e = ExtractGlobals.size(); i != e; ++i) {
GlobalValue *GV = M.get()->getNamedGlobal(ExtractGlobals[i]);
if (!GV) {
errs() << argv[0] << ": program doesn't contain global named '"
<< ExtractGlobals[i] << "'!\n";
return 1;
}
GVs.insert(GV);
}
// Extract globals via regular expression matching.
for (size_t i = 0, e = ExtractRegExpGlobals.size(); i != e; ++i) {
std::string Error;
Regex RegEx(ExtractRegExpGlobals[i]);
if (!RegEx.isValid(Error)) {
errs() << argv[0] << ": '" << ExtractRegExpGlobals[i] << "' "
"invalid regex: " << Error;
}
bool match = false;
for (Module::global_iterator GV = M.get()->global_begin(),
E = M.get()->global_end(); GV != E; GV++) {
if (RegEx.match(GV->getName())) {
GVs.insert(&*GV);
match = true;
}
}
if (!match) {
errs() << argv[0] << ": program doesn't contain global named '"
<< ExtractRegExpGlobals[i] << "'!\n";
return 1;
}
}
// Figure out which functions we should extract.
for (size_t i = 0, e = ExtractFuncs.size(); i != e; ++i) {
GlobalValue *GV = M.get()->getFunction(ExtractFuncs[i]);
if (!GV) {
errs() << argv[0] << ": program doesn't contain function named '"
<< ExtractFuncs[i] << "'!\n";
return 1;
}
GVs.insert(GV);
}
// Extract functions via regular expression matching.
for (size_t i = 0, e = ExtractRegExpFuncs.size(); i != e; ++i) {
std::string Error;
StringRef RegExStr = ExtractRegExpFuncs[i];
Regex RegEx(RegExStr);
if (!RegEx.isValid(Error)) {
errs() << argv[0] << ": '" << ExtractRegExpFuncs[i] << "' "
"invalid regex: " << Error;
}
bool match = false;
for (Module::iterator F = M.get()->begin(), E = M.get()->end(); F != E;
F++) {
if (RegEx.match(F->getName())) {
GVs.insert(&*F);
match = true;
}
}
if (!match) {
errs() << argv[0] << ": program doesn't contain global named '"
<< ExtractRegExpFuncs[i] << "'!\n";
return 1;
}
}
// Materialize requisite global values.
if (!DeleteFn)
for (size_t i = 0, e = GVs.size(); i != e; ++i) {
GlobalValue *GV = GVs[i];
if (GV->isMaterializable()) {
std::string ErrInfo;
if (GV->Materialize(&ErrInfo)) {
errs() << argv[0] << ": error reading input: " << ErrInfo << "\n";
return 1;
}
}
}
else {
// Deleting. Materialize every GV that's *not* in GVs.
SmallPtrSet<GlobalValue *, 8> GVSet(GVs.begin(), GVs.end());
for (Module::global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *G = I;
if (!GVSet.count(G) && G->isMaterializable()) {
std::string ErrInfo;
if (G->Materialize(&ErrInfo)) {
errs() << argv[0] << ": error reading input: " << ErrInfo << "\n";
return 1;
}
}
}
for (Module::iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = I;
if (!GVSet.count(F) && F->isMaterializable()) {
std::string ErrInfo;
if (F->Materialize(&ErrInfo)) {
errs() << argv[0] << ": error reading input: " << ErrInfo << "\n";
return 1;
}
}
}
}
// In addition to deleting all other functions, we also want to spiff it
// up a little bit. Do this now.
PassManager Passes;
Passes.add(new TargetData(M.get())); // Use correct TargetData
std::vector<GlobalValue*> Gvs(GVs.begin(), GVs.end());
Passes.add(createGVExtractionPass(Gvs, DeleteFn));
if (!DeleteFn)
Passes.add(createGlobalDCEPass()); // Delete unreachable globals
Passes.add(createStripDeadDebugInfoPass()); // Remove dead debug info
Passes.add(createStripDeadPrototypesPass()); // Remove dead func decls
std::string ErrorInfo;
tool_output_file Out(OutputFilename.c_str(), ErrorInfo,
raw_fd_ostream::F_Binary);
if (!ErrorInfo.empty()) {
errs() << ErrorInfo << '\n';
return 1;
}
if (OutputAssembly)
Passes.add(createPrintModulePass(&Out.os()));
else if (Force || !CheckBitcodeOutputToConsole(Out.os(), true))
Passes.add(createBitcodeWriterPass(Out.os()));
Passes.run(*M.get());
// Declare success.
Out.keep();
return 0;
}
void HeterotbbTransform::rewrite_CPP(Module &M) {
// Collect initial set of hetero functions
vector<Instruction *> toDelete;
DenseMap<Function*, Function *> FunctionMap[2];
templat = NULL;
templat_join = NULL;
for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
//if (isa<UnaryInst>(I))
if (!I->isDeclaration()) {
//DEBUG(dbgs() << "Func"<<*I<<"\n\n");
for (Function::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE; ++BBI) {
for (BasicBlock::iterator INSNI = BBI->begin(), INSNE = BBI->end(); INSNI != INSNE; ++INSNI) {
if (isa<CallInst>(INSNI) || isa<InvokeInst>(INSNI)) {
//DEBUG(dbgs()<<*INSNI<<"\n");
CallSite CI(cast<Instruction>(INSNI));
// errs()<<"INS"<<INSNI<<"\n";
if (CI.getCalledFunction() == NULL) continue;
int type =0;
type = is_hetero_function_f(M, CI);//return a 1 for parallel_for and 2 for parallel_reduce
if (type) {
//CI->dump();
DEBUG(dbgs() << type<<":Hetero_fun "<<CI.getCalledFunction()->getName()<<"\n");
//get the kernel body
Function *f= get_hetero_func(M,CI);
//@_ZNK12diff_testingclEi
if(f!=NULL)//should never be null if the body is called.
{
DEBUG(dbgs() << " Kernel: "<<f->getName()<<"\n");
//create new function for the kernel
Function *nf=NULL;
if(FunctionMap[type-1].find(f)==FunctionMap[type-1].end()) {
nf = write_new_hetero_kernel(M,f,type);
FunctionMap[type-1][f]=nf;
DEBUG(dbgs() << " New Kernel Created: "<<nf->getName()<<" created\n\n");
/* Added by Raj to add inline attributes recursively*/
add_inline_attributes(nf);
}
else {
nf=FunctionMap[type-1][f];
DEBUG(dbgs() << " Kernel Exists: "<<nf->getName()<<" created\n\n");
}
//rewrite the hetero call site to offload
if(isa<CallInst>(INSNI))
rewrite_call_site(M,CI,nf,type);
else if(isa<InvokeInst>(INSNI)) //must be invoke
rewrite_invoke_site(M,CI,nf,type);
else {
DEBUG(dbgs()<<"ERROR\n");
}
//delete the old call site
toDelete.push_back(INSNI);
}
else {
DEBUG(dbgs() << " Parallel for/reduce hetero do not call any functions\n\n");
}
}
/*
#ifdef HETERO_GCD_H
else if(hetero_function = get_hetero_function(M, CI)) {
entry_hetero_function_set.insert(hetero_function);
#ifdef HETERO_GCD_ALLOC_TO_MALLOC
block2insnMap.insert(pair<Function *, Instruction *>(hetero_function, INSNI));
#endif
}
#endif
*/
}
}
}
}
}
while(!toDelete.empty()) {
Instruction *g = toDelete.back();
toDelete.pop_back();
// g->replaceAllUsesWith(UndefValue::get(g->getType()));
g->eraseFromParent();
}
/* delete the template functions */
if (templat_join != NULL) {
templat_join->replaceAllUsesWith(UndefValue::get(templat_join->getType()));
templat_join->eraseFromParent();
}
if (templat != NULL) {
templat->replaceAllUsesWith(UndefValue::get(templat->getType()));
templat->eraseFromParent();
}
//erase the annotation
GlobalVariable *annot = M.getGlobalVariable("opencl_metadata");
if (annot != NULL) annot->eraseFromParent();
annot = M.getGlobalVariable("opencl_metadata_type");
if (annot != NULL) annot->eraseFromParent();
annot = M.getGlobalVariable("opencl_kernel_join_name_locals");
if (annot != NULL) annot->eraseFromParent();
annot = M.getGlobalVariable("opencl_kernel_join_name_parameters");
if (annot != NULL) annot->eraseFromParent();
annot = M.getGlobalVariable("opencl_kernel_join_name_local_arr");
if (annot != NULL) annot->eraseFromParent();
}
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
GlobalVariable *Name = Inc->getName();
auto It = RegionCounters.find(Name);
if (It != RegionCounters.end())
return It->second;
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
// only one copy of counters of the COMDAT function will be emitted after
// linking.
Function *Fn = Inc->getParent()->getParent();
Comdat *ProfileVarsComdat = nullptr;
if (Fn->hasComdat())
ProfileVarsComdat = M->getOrInsertComdat(StringRef(getVarName(Inc, "vars")));
Name->setSection(getNameSection());
Name->setAlignment(1);
Name->setComdat(ProfileVarsComdat);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
Constant::getNullValue(CounterTy),
getVarName(Inc, "counters"));
Counters->setVisibility(Name->getVisibility());
Counters->setSection(getCountersSection());
Counters->setAlignment(8);
Counters->setComdat(ProfileVarsComdat);
RegionCounters[Inc->getName()] = Counters;
// Create data variable.
auto *NameArrayTy = Name->getType()->getPointerElementType();
auto *Int32Ty = Type::getInt32Ty(Ctx);
auto *Int64Ty = Type::getInt64Ty(Ctx);
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Constant *DataVals[] = {
ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
ConstantInt::get(Int32Ty, NumCounters),
ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
ConstantExpr::getBitCast(Name, Int8PtrTy),
ConstantExpr::getBitCast(Counters, Int64PtrTy)};
auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
getVarName(Inc, "data"));
Data->setVisibility(Name->getVisibility());
Data->setSection(getDataSection());
Data->setAlignment(8);
Data->setComdat(ProfileVarsComdat);
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
return Counters;
}
Function* HeterotbbTransform::create_new_join(Module &M,Function *join) {
//reduce->dump();
if(!templat) {
DEBUG(dbgs()<<"NO Template Found\n");
return NULL;
}
//join_main->dump();
DEBUG(dbgs()<<"Objext size array"<<object_size_hetero<<"\n");
//create a global with 64*object/4 size
GlobalVariable *gb = M.getGlobalVariable("opencl_kernel_join_name_local_arr",true);
//gb->dump();
Value *val=gb->getOperand(0);
//if(isa<ArrayType>(val->getType()))DEBUG(dbgs()<<"YES\n");
//since we are creating an integer array, the size gets divided by 4
// Do not make it a global variable -- make it a local variable with annotation for local
int local_size = 64*object_size_hetero;
/*const*/ ArrayType *arr= ArrayType::get(Type::getInt32Ty(M.getContext()),(64*object_size_hetero)/4);
/*vector<Constant *> Initializer;
APInt zero(32,0);
for(int i=0;i<(16*object_size_hetero);i++){
Initializer.push_back(ConstantInt::get(M.getContext(),zero));
}
Constant *init = ConstantArray::get(arr, Initializer);
GlobalVariable *new_gb = new GlobalVariable(M, arr, false, GlobalVariable::InternalLinkage,init, "__hetero_local_"+join->getName()+"__local__arr",gb,false,3);
new_gb->setAlignment(gb->getAlignment());
DEBUG(dbgs()<<"Global Created\n");
new_gb->dump();
*/
vector</*const*/ Type *> params;
int temp_size=0;
object_size_hetero=0;
// void join(class.name *,class.name *)
//re-write join
const FunctionType *FTy = join->getFunctionType();
Function::arg_iterator ArgI = join->arg_begin();
// class.name *
params.push_back(PointerType::get((dyn_cast<PointerType>(ArgI->getType())->getElementType()),3));
params.push_back(PointerType::get((dyn_cast<PointerType>(ArgI->getType())->getElementType()),3));
/*const*/ Type *RetTy = FTy->getReturnType();
FunctionType *NFty = FunctionType::get(RetTy,params, false);
Function *NF=Function::Create(NFty, join->getLinkage(), join->getName()+"_inline");
NF->copyAttributesFrom(join);
#if EXPLICIT_REWRITE
copy_function(NF,join);
#else
ValueToValueMapTy VMap;
for(Function::arg_iterator FI = join->arg_begin(), FE=join->arg_end(), DI=NF->arg_begin(); FE!=FI; ++FI,++DI) {
DI->setName(FI->getName());
VMap[FI]=DI;
}
CloneFunctionWithExistingBBInto(NF, NULL, join, VMap);
#endif
//NF->removeFnAttr(Attributes::get(NF->getContext(), Attribute::NoInline));
NF->addFnAttr(Attribute::AlwaysInline);
join->getParent()->getFunctionList().insert(join, NF);
params.clear();
const FunctionType *FTemp = templat->getFunctionType();
//create a new template
for(Function::arg_iterator FI = templat->arg_begin(), FE=templat->arg_end(); FE!=FI; ++FI) {
params.push_back(FI->getType());
}
// templat->replaceUsesOfWith(reduce,NF);
RetTy = FTy->getReturnType();
NFty = FunctionType::get(RetTy,params, false);
Function *templat_copy =Function::Create(NFty, join->getLinkage(), join->getName()+"_hetero");
templat_copy->copyAttributesFrom(templat);
#if EXPLICIT_REWRITE
copy_function(templat_copy,templat);
#else
ValueToValueMapTy VMapp;
for(Function::arg_iterator FI = templat->arg_begin(), FE=templat->arg_end(), DI=templat_copy->arg_begin(); FE!=FI; ++FI,++DI) {
DI->setName(FI->getName());
VMapp[FI]=DI;
}
CloneFunctionWithExistingBBInto(templat_copy, NULL, templat, VMapp);
#endif
/* create a local variable with the following type */
Function::iterator BI = templat_copy->begin();
BasicBlock::iterator II = BI->begin();
Instruction *insn = &(*II);
Constant *l_size = ConstantInt::get(Type::getInt32Ty(M.getContext()), local_size);
Instruction *new_gb_ = new AllocaInst(arr, l_size, gb->getAlignment(), "hetero_local", insn);
//new_gb_->dump();
Value *Elts[] = {MDString::get(M.getContext(), new_gb_->getName())};
MDNode *Node = MDNode::get(M.getContext(), Elts);
new_gb_->setMetadata("local",Node);
Instruction *new_gb= CastInst::Create(Instruction::BitCast, new_gb_,
PointerType::get(arr,3), "hetero_local_cast", insn);
//new_gb->dump();
Value *Elts1[] = {MDString::get(M.getContext(), new_gb->getName())};
MDNode *Node1 = MDNode::get(M.getContext(), Elts1);
new_gb->setMetadata("local_cast",Node1);
edit_template_function(M,templat_copy,NF,gb,new_gb);
templat->getParent()->getFunctionList().insert(templat, templat_copy);
return templat_copy;
}
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
GlobalVariable *NamePtr = Inc->getName();
auto It = ProfileDataMap.find(NamePtr);
PerFunctionProfileData PD;
if (It != ProfileDataMap.end()) {
if (It->second.RegionCounters)
return It->second.RegionCounters;
PD = It->second;
}
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
// only one copy of counters of the COMDAT function will be emitted after
// linking.
Function *Fn = Inc->getParent()->getParent();
Comdat *ProfileVarsComdat = nullptr;
ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
auto *CounterPtr =
new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
Constant::getNullValue(CounterTy),
getVarName(Inc, getInstrProfCountersVarPrefix()));
CounterPtr->setVisibility(NamePtr->getVisibility());
CounterPtr->setSection(
getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
CounterPtr->setAlignment(8);
CounterPtr->setComdat(ProfileVarsComdat);
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
// Allocate statically the array of pointers to value profile nodes for
// the current function.
Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
uint64_t NS = 0;
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
NS += PD.NumValueSites[Kind];
if (NS) {
ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
auto *ValuesVar =
new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
Constant::getNullValue(ValuesTy),
getVarName(Inc, getInstrProfValuesVarPrefix()));
ValuesVar->setVisibility(NamePtr->getVisibility());
ValuesVar->setSection(
getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
ValuesVar->setAlignment(8);
ValuesVar->setComdat(ProfileVarsComdat);
ValuesPtrExpr =
ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
}
}
// Create data variable.
auto *Int16Ty = Type::getInt16Ty(Ctx);
auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
Type *DataTypes[] = {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
? ConstantExpr::getBitCast(Fn, Int8PtrTy)
: ConstantPointerNull::get(Int8PtrTy);
Constant *Int16ArrayVals[IPVK_Last + 1];
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
Constant *DataVals[] = {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
getVarName(Inc, getInstrProfDataVarPrefix()));
Data->setVisibility(NamePtr->getVisibility());
Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
Data->setComdat(ProfileVarsComdat);
PD.RegionCounters = CounterPtr;
PD.DataVar = Data;
ProfileDataMap[NamePtr] = PD;
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
// Now that the linkage set by the FE has been passed to the data and counter
// variables, reset Name variable's linkage and visibility to private so that
// it can be removed later by the compiler.
NamePtr->setLinkage(GlobalValue::PrivateLinkage);
// Collect the referenced names to be used by emitNameData.
ReferencedNames.push_back(NamePtr);
return CounterPtr;
}
bool GenericToNVVM::runOnModule(Module &M) {
// Create a clone of each global variable that has the default address space.
// The clone is created with the global address space specifier, and the pair
// of original global variable and its clone is placed in the GVMap for later
// use.
for (Module::global_iterator I = M.global_begin(), E = M.global_end();
I != E;) {
GlobalVariable *GV = &*I++;
if (GV->getType()->getAddressSpace() == llvm::ADDRESS_SPACE_GENERIC &&
!llvm::isTexture(*GV) && !llvm::isSurface(*GV) &&
!llvm::isSampler(*GV) && !GV->getName().startswith("llvm.")) {
GlobalVariable *NewGV = new GlobalVariable(
M, GV->getValueType(), GV->isConstant(),
GV->getLinkage(),
GV->hasInitializer() ? GV->getInitializer() : nullptr,
"", GV, GV->getThreadLocalMode(), llvm::ADDRESS_SPACE_GLOBAL);
NewGV->copyAttributesFrom(GV);
GVMap[GV] = NewGV;
}
}
// Return immediately, if every global variable has a specific address space
// specifier.
if (GVMap.empty()) {
return false;
}
// Walk through the instructions in function defitinions, and replace any use
// of original global variables in GVMap with a use of the corresponding
// copies in GVMap. If necessary, promote constants to instructions.
for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
if (I->isDeclaration()) {
continue;
}
IRBuilder<> Builder(I->getEntryBlock().getFirstNonPHIOrDbg());
for (Function::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE;
++BBI) {
for (BasicBlock::iterator II = BBI->begin(), IE = BBI->end(); II != IE;
++II) {
for (unsigned i = 0, e = II->getNumOperands(); i < e; ++i) {
Value *Operand = II->getOperand(i);
if (isa<Constant>(Operand)) {
II->setOperand(
i, remapConstant(&M, &*I, cast<Constant>(Operand), Builder));
}
}
}
}
ConstantToValueMap.clear();
}
// Copy GVMap over to a standard value map.
ValueToValueMapTy VM;
for (auto I = GVMap.begin(), E = GVMap.end(); I != E; ++I)
VM[I->first] = I->second;
// Walk through the metadata section and update the debug information
// associated with the global variables in the default address space.
for (NamedMDNode &I : M.named_metadata()) {
remapNamedMDNode(VM, &I);
}
// Walk through the global variable initializers, and replace any use of
// original global variables in GVMap with a use of the corresponding copies
// in GVMap. The copies need to be bitcast to the original global variable
// types, as we cannot use cvta in global variable initializers.
for (GVMapTy::iterator I = GVMap.begin(), E = GVMap.end(); I != E;) {
GlobalVariable *GV = I->first;
GlobalVariable *NewGV = I->second;
// Remove GV from the map so that it can be RAUWed. Note that
// DenseMap::erase() won't invalidate any iterators but this one.
auto Next = std::next(I);
GVMap.erase(I);
I = Next;
Constant *BitCastNewGV = ConstantExpr::getPointerCast(NewGV, GV->getType());
// At this point, the remaining uses of GV should be found only in global
// variable initializers, as other uses have been already been removed
// while walking through the instructions in function definitions.
GV->replaceAllUsesWith(BitCastNewGV);
std::string Name = GV->getName();
GV->eraseFromParent();
NewGV->setName(Name);
}
assert(GVMap.empty() && "Expected it to be empty by now");
return true;
}
bool ConstantMerge::runOnModule(Module &M) {
TD = getAnalysisIfAvailable<TargetData>();
// Find all the globals that are marked "used". These cannot be merged.
SmallPtrSet<const GlobalValue*, 8> UsedGlobals;
FindUsedValues(M.getGlobalVariable("llvm.used"), UsedGlobals);
FindUsedValues(M.getGlobalVariable("llvm.compiler.used"), UsedGlobals);
// Map unique <constants, has-unknown-alignment> pairs to globals. We don't
// want to merge globals of unknown alignment with those of explicit
// alignment. If we have TargetData, we always know the alignment.
DenseMap<PointerIntPair<Constant*, 1, bool>, GlobalVariable*> CMap;
// Replacements - This vector contains a list of replacements to perform.
SmallVector<std::pair<GlobalVariable*, GlobalVariable*>, 32> Replacements;
bool MadeChange = false;
// Iterate constant merging while we are still making progress. Merging two
// constants together may allow us to merge other constants together if the
// second level constants have initializers which point to the globals that
// were just merged.
while (1) {
// First: Find the canonical constants others will be merged with.
for (Module::global_iterator GVI = M.global_begin(), E = M.global_end();
GVI != E; ) {
GlobalVariable *GV = GVI++;
// If this GV is dead, remove it.
GV->removeDeadConstantUsers();
if (GV->use_empty() && GV->hasLocalLinkage()) {
GV->eraseFromParent();
continue;
}
// Only process constants with initializers in the default address space.
if (!GV->isConstant() || !GV->hasDefinitiveInitializer() ||
GV->getType()->getAddressSpace() != 0 || GV->hasSection() ||
// Don't touch values marked with attribute(used).
UsedGlobals.count(GV))
continue;
// This transformation is legal for weak ODR globals in the sense it
// doesn't change semantics, but we really don't want to perform it
// anyway; it's likely to pessimize code generation, and some tools
// (like the Darwin linker in cases involving CFString) don't expect it.
if (GV->isWeakForLinker())
continue;
Constant *Init = GV->getInitializer();
// Check to see if the initializer is already known.
PointerIntPair<Constant*, 1, bool> Pair(Init, hasKnownAlignment(GV));
GlobalVariable *&Slot = CMap[Pair];
// If this is the first constant we find or if the old one is local,
// replace with the current one. If the current is externally visible
// it cannot be replace, but can be the canonical constant we merge with.
if (Slot == 0 || IsBetterCannonical(*GV, *Slot))
Slot = GV;
}
// Second: identify all globals that can be merged together, filling in
// the Replacements vector. We cannot do the replacement in this pass
// because doing so may cause initializers of other globals to be rewritten,
// invalidating the Constant* pointers in CMap.
for (Module::global_iterator GVI = M.global_begin(), E = M.global_end();
GVI != E; ) {
GlobalVariable *GV = GVI++;
// Only process constants with initializers in the default address space.
if (!GV->isConstant() || !GV->hasDefinitiveInitializer() ||
GV->getType()->getAddressSpace() != 0 || GV->hasSection() ||
// Don't touch values marked with attribute(used).
UsedGlobals.count(GV))
continue;
// We can only replace constant with local linkage.
if (!GV->hasLocalLinkage())
continue;
Constant *Init = GV->getInitializer();
// Check to see if the initializer is already known.
PointerIntPair<Constant*, 1, bool> Pair(Init, hasKnownAlignment(GV));
GlobalVariable *Slot = CMap[Pair];
if (!Slot || Slot == GV)
continue;
if (!Slot->hasUnnamedAddr() && !GV->hasUnnamedAddr())
continue;
if (!GV->hasUnnamedAddr())
Slot->setUnnamedAddr(false);
// Make all uses of the duplicate constant use the canonical version.
Replacements.push_back(std::make_pair(GV, Slot));
}
if (Replacements.empty())
return MadeChange;
CMap.clear();
// Now that we have figured out which replacements must be made, do them all
// now. This avoid invalidating the pointers in CMap, which are unneeded
// now.
for (unsigned i = 0, e = Replacements.size(); i != e; ++i) {
// Bump the alignment if necessary.
if (Replacements[i].first->getAlignment() ||
Replacements[i].second->getAlignment()) {
Replacements[i].second->setAlignment(std::max(
Replacements[i].first->getAlignment(),
Replacements[i].second->getAlignment()));
}
// Eliminate any uses of the dead global.
Replacements[i].first->replaceAllUsesWith(Replacements[i].second);
// Delete the global value from the module.
assert(Replacements[i].first->hasLocalLinkage() &&
"Refusing to delete an externally visible global variable.");
Replacements[i].first->eraseFromParent();
}
NumMerged += Replacements.size();
Replacements.clear();
}
}
bool ConstantMerge::runOnModule(Module &M) {
// Find all the globals that are marked "used". These cannot be merged.
SmallPtrSet<const GlobalValue*, 8> UsedGlobals;
FindUsedValues(M.getGlobalVariable("llvm.used"), UsedGlobals);
FindUsedValues(M.getGlobalVariable("llvm.compiler.used"), UsedGlobals);
// Map unique constant/section pairs to globals. We don't want to merge
// globals in different sections.
DenseMap<Constant*, GlobalVariable*> CMap;
// Replacements - This vector contains a list of replacements to perform.
SmallVector<std::pair<GlobalVariable*, GlobalVariable*>, 32> Replacements;
bool MadeChange = false;
// Iterate constant merging while we are still making progress. Merging two
// constants together may allow us to merge other constants together if the
// second level constants have initializers which point to the globals that
// were just merged.
while (1) {
// First pass: identify all globals that can be merged together, filling in
// the Replacements vector. We cannot do the replacement in this pass
// because doing so may cause initializers of other globals to be rewritten,
// invalidating the Constant* pointers in CMap.
//
for (Module::global_iterator GVI = M.global_begin(), E = M.global_end();
GVI != E; ) {
GlobalVariable *GV = GVI++;
// If this GV is dead, remove it.
GV->removeDeadConstantUsers();
if (GV->use_empty() && GV->hasLocalLinkage()) {
GV->eraseFromParent();
continue;
}
// Only process constants with initializers in the default addres space.
if (!GV->isConstant() ||!GV->hasDefinitiveInitializer() ||
GV->getType()->getAddressSpace() != 0 || !GV->getSection().empty() ||
// Don't touch values marked with attribute(used).
UsedGlobals.count(GV))
continue;
Constant *Init = GV->getInitializer();
// Check to see if the initializer is already known.
GlobalVariable *&Slot = CMap[Init];
if (Slot == 0) { // Nope, add it to the map.
Slot = GV;
} else if (GV->hasLocalLinkage()) { // Yup, this is a duplicate!
// Make all uses of the duplicate constant use the canonical version.
Replacements.push_back(std::make_pair(GV, Slot));
}
}
if (Replacements.empty())
return MadeChange;
CMap.clear();
// Now that we have figured out which replacements must be made, do them all
// now. This avoid invalidating the pointers in CMap, which are unneeded
// now.
for (unsigned i = 0, e = Replacements.size(); i != e; ++i) {
// Eliminate any uses of the dead global.
Replacements[i].first->replaceAllUsesWith(Replacements[i].second);
// Delete the global value from the module.
Replacements[i].first->eraseFromParent();
}
NumMerged += Replacements.size();
Replacements.clear();
}
}
bool GCOVProfiler::emitProfileArcs() {
NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
if (!CU_Nodes) return false;
bool Result = false;
bool InsertIndCounterIncrCode = false;
for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i));
SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
for (auto *SP : CU->getSubprograms()) {
Function *F = FnMap[SP];
if (!F) continue;
if (!functionHasLines(F)) continue;
if (!Result) Result = true;
unsigned Edges = 0;
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
TerminatorInst *TI = BB->getTerminator();
if (isa<ReturnInst>(TI))
++Edges;
else
Edges += TI->getNumSuccessors();
}
ArrayType *CounterTy =
ArrayType::get(Type::getInt64Ty(*Ctx), Edges);
GlobalVariable *Counters =
new GlobalVariable(*M, CounterTy, false,
GlobalValue::InternalLinkage,
Constant::getNullValue(CounterTy),
"__llvm_gcov_ctr");
CountersBySP.push_back(std::make_pair(Counters, SP));
UniqueVector<BasicBlock *> ComplexEdgePreds;
UniqueVector<BasicBlock *> ComplexEdgeSuccs;
unsigned Edge = 0;
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
TerminatorInst *TI = BB->getTerminator();
int Successors = isa<ReturnInst>(TI) ? 1 : TI->getNumSuccessors();
if (Successors) {
if (Successors == 1) {
IRBuilder<> Builder(&*BB->getFirstInsertionPt());
Value *Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0,
Edge);
Value *Count = Builder.CreateLoad(Counter);
Count = Builder.CreateAdd(Count, Builder.getInt64(1));
Builder.CreateStore(Count, Counter);
} else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
IRBuilder<> Builder(BI);
Value *Sel = Builder.CreateSelect(BI->getCondition(),
Builder.getInt64(Edge),
Builder.getInt64(Edge + 1));
SmallVector<Value *, 2> Idx;
Idx.push_back(Builder.getInt64(0));
Idx.push_back(Sel);
Value *Counter = Builder.CreateInBoundsGEP(Counters->getValueType(),
Counters, Idx);
Value *Count = Builder.CreateLoad(Counter);
Count = Builder.CreateAdd(Count, Builder.getInt64(1));
Builder.CreateStore(Count, Counter);
} else {
ComplexEdgePreds.insert(&*BB);
for (int i = 0; i != Successors; ++i)
ComplexEdgeSuccs.insert(TI->getSuccessor(i));
}
Edge += Successors;
}
}
if (!ComplexEdgePreds.empty()) {
GlobalVariable *EdgeTable =
buildEdgeLookupTable(F, Counters,
ComplexEdgePreds, ComplexEdgeSuccs);
GlobalVariable *EdgeState = getEdgeStateValue();
for (int i = 0, e = ComplexEdgePreds.size(); i != e; ++i) {
IRBuilder<> Builder(&*ComplexEdgePreds[i + 1]->getFirstInsertionPt());
Builder.CreateStore(Builder.getInt32(i), EdgeState);
}
for (int i = 0, e = ComplexEdgeSuccs.size(); i != e; ++i) {
// Call runtime to perform increment.
IRBuilder<> Builder(&*ComplexEdgeSuccs[i + 1]->getFirstInsertionPt());
Value *CounterPtrArray =
Builder.CreateConstInBoundsGEP2_64(EdgeTable, 0,
i * ComplexEdgePreds.size());
// Build code to increment the counter.
InsertIndCounterIncrCode = true;
Builder.CreateCall(getIncrementIndirectCounterFunc(),
{EdgeState, CounterPtrArray});
}
}
}
Function *WriteoutF = insertCounterWriteout(CountersBySP);
Function *FlushF = insertFlush(CountersBySP);
// Create a small bit of code that registers the "__llvm_gcov_writeout" to
// be executed at exit and the "__llvm_gcov_flush" function to be executed
// when "__gcov_flush" is called.
FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
Function *F = Function::Create(FTy, GlobalValue::InternalLinkage,
"__llvm_gcov_init", M);
F->setUnnamedAddr(true);
F->setLinkage(GlobalValue::InternalLinkage);
F->addFnAttr(Attribute::NoInline);
if (Options.NoRedZone)
F->addFnAttr(Attribute::NoRedZone);
BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F);
IRBuilder<> Builder(BB);
FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
Type *Params[] = {
PointerType::get(FTy, 0),
PointerType::get(FTy, 0)
};
FTy = FunctionType::get(Builder.getVoidTy(), Params, false);
// Initialize the environment and register the local writeout and flush
// functions.
Constant *GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
Builder.CreateCall(GCOVInit, {WriteoutF, FlushF});
Builder.CreateRetVoid();
appendToGlobalCtors(*M, F, 0);
}
if (InsertIndCounterIncrCode)
insertIndirectCounterIncrement();
return Result;
}
bool SanitizerCoverageModule::runOnModule(Module &M) {
if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
return false;
C = &(M.getContext());
DL = &M.getDataLayout();
CurModule = &M;
IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
Type *VoidTy = Type::getVoidTy(*C);
IRBuilder<> IRB(*C);
Type *Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty());
Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
Int64PtrTy = PointerType::getUnqual(IRB.getInt64Ty());
Int64Ty = IRB.getInt64Ty();
SanCovFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovName, VoidTy, Int32PtrTy, nullptr));
SanCovWithCheckFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovWithCheckName, VoidTy, Int32PtrTy, nullptr));
SanCovTracePCIndir =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovTracePCIndir, VoidTy, IntptrTy, nullptr));
SanCovIndirCallFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovIndirCallName, VoidTy, IntptrTy, IntptrTy, nullptr));
SanCovTraceCmpFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovTraceCmp, VoidTy, Int64Ty, Int64Ty, Int64Ty, nullptr));
SanCovTraceSwitchFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovTraceSwitch, VoidTy, Int64Ty, Int64PtrTy, nullptr));
// We insert an empty inline asm after cov callbacks to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
/*hasSideEffects=*/true);
SanCovTracePC = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovTracePC, VoidTy, nullptr));
SanCovTraceEnter = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovTraceEnter, VoidTy, Int32PtrTy, nullptr));
SanCovTraceBB = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovTraceBB, VoidTy, Int32PtrTy, nullptr));
// At this point we create a dummy array of guards because we don't
// know how many elements we will need.
Type *Int32Ty = IRB.getInt32Ty();
Type *Int8Ty = IRB.getInt8Ty();
GuardArray =
new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
if (Options.Use8bitCounters)
EightBitCounterArray =
new GlobalVariable(M, Int8Ty, false, GlobalVariable::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
for (auto &F : M)
runOnFunction(F);
auto N = NumberOfInstrumentedBlocks();
// Now we know how many elements we need. Create an array of guards
// with one extra element at the beginning for the size.
Type *Int32ArrayNTy = ArrayType::get(Int32Ty, N + 1);
GlobalVariable *RealGuardArray = new GlobalVariable(
M, Int32ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int32ArrayNTy), "__sancov_gen_cov");
// Replace the dummy array with the real one.
GuardArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealGuardArray, Int32PtrTy));
GuardArray->eraseFromParent();
GlobalVariable *RealEightBitCounterArray;
if (Options.Use8bitCounters) {
// Make sure the array is 16-aligned.
static const int kCounterAlignment = 16;
Type *Int8ArrayNTy = ArrayType::get(Int8Ty, alignTo(N, kCounterAlignment));
RealEightBitCounterArray = new GlobalVariable(
M, Int8ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int8ArrayNTy), "__sancov_gen_cov_counter");
RealEightBitCounterArray->setAlignment(kCounterAlignment);
EightBitCounterArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy));
EightBitCounterArray->eraseFromParent();
}
// Create variable for module (compilation unit) name
Constant *ModNameStrConst =
ConstantDataArray::getString(M.getContext(), M.getName(), true);
GlobalVariable *ModuleName =
new GlobalVariable(M, ModNameStrConst->getType(), true,
GlobalValue::PrivateLinkage, ModNameStrConst);
Function *CtorFunc;
std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
M, kSanCovModuleCtorName, kSanCovModuleInitName,
{Int32PtrTy, IntptrTy, Int8PtrTy, Int8PtrTy},
{IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
ConstantInt::get(IntptrTy, N),
Options.Use8bitCounters
? IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy)
: Constant::getNullValue(Int8PtrTy),
IRB.CreatePointerCast(ModuleName, Int8PtrTy)});
appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
return true;
}
bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
Module &M, bool isConst, unsigned AddrSpace) const {
// FIXME: Find better heuristics
std::stable_sort(Globals.begin(), Globals.end(),
[this](const GlobalVariable *GV1, const GlobalVariable *GV2) {
Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
Type *Ty2 = cast<PointerType>(GV2->getType())->getElementType();
return (DL->getTypeAllocSize(Ty1) < DL->getTypeAllocSize(Ty2));
});
Type *Int32Ty = Type::getInt32Ty(M.getContext());
assert(Globals.size() > 1);
// FIXME: This simple solution merges globals all together as maximum as
// possible. However, with this solution it would be hard to remove dead
// global symbols at link-time. An alternative solution could be checking
// global symbols references function by function, and make the symbols
// being referred in the same function merged and we would probably need
// to introduce heuristic algorithm to solve the merge conflict from
// different functions.
for (size_t i = 0, e = Globals.size(); i != e; ) {
size_t j = 0;
uint64_t MergedSize = 0;
std::vector<Type*> Tys;
std::vector<Constant*> Inits;
bool HasExternal = false;
GlobalVariable *TheFirstExternal = 0;
for (j = i; j != e; ++j) {
Type *Ty = Globals[j]->getType()->getElementType();
MergedSize += DL->getTypeAllocSize(Ty);
if (MergedSize > MaxOffset) {
break;
}
Tys.push_back(Ty);
Inits.push_back(Globals[j]->getInitializer());
if (Globals[j]->hasExternalLinkage() && !HasExternal) {
HasExternal = true;
TheFirstExternal = Globals[j];
}
}
// If merged variables doesn't have external linkage, we needn't to expose
// the symbol after merging.
GlobalValue::LinkageTypes Linkage = HasExternal
? GlobalValue::ExternalLinkage
: GlobalValue::InternalLinkage;
StructType *MergedTy = StructType::get(M.getContext(), Tys);
Constant *MergedInit = ConstantStruct::get(MergedTy, Inits);
// If merged variables have external linkage, we use symbol name of the
// first variable merged as the suffix of global symbol name. This would
// be able to avoid the link-time naming conflict for globalm symbols.
GlobalVariable *MergedGV = new GlobalVariable(
M, MergedTy, isConst, Linkage, MergedInit,
HasExternal ? "_MergedGlobals_" + TheFirstExternal->getName()
: "_MergedGlobals",
nullptr, GlobalVariable::NotThreadLocal, AddrSpace);
for (size_t k = i; k < j; ++k) {
GlobalValue::LinkageTypes Linkage = Globals[k]->getLinkage();
std::string Name = Globals[k]->getName();
Constant *Idx[2] = {
ConstantInt::get(Int32Ty, 0),
ConstantInt::get(Int32Ty, k-i)
};
Constant *GEP = ConstantExpr::getInBoundsGetElementPtr(MergedGV, Idx);
Globals[k]->replaceAllUsesWith(GEP);
Globals[k]->eraseFromParent();
if (Linkage != GlobalValue::InternalLinkage) {
// Generate a new alias...
auto *PTy = cast<PointerType>(GEP->getType());
GlobalAlias::create(PTy->getElementType(), PTy->getAddressSpace(),
Linkage, Name, GEP, &M);
}
NumMerged++;
}
i = j;
}
return true;
}
/// addDefinedSymbol - Add a defined symbol to the list.
void LTOModule::addDefinedSymbol(GlobalValue *def, bool isFunction) {
// ignore all llvm.* symbols
if (def->getName().startswith("llvm."))
return;
// string is owned by _defines
SmallString<64> Buffer;
_mangler.getNameWithPrefix(Buffer, def, false);
// set alignment part log2() can have rounding errors
uint32_t align = def->getAlignment();
uint32_t attr = align ? CountTrailingZeros_32(def->getAlignment()) : 0;
// set permissions part
if (isFunction) {
attr |= LTO_SYMBOL_PERMISSIONS_CODE;
} else {
GlobalVariable *gv = dyn_cast<GlobalVariable>(def);
if (gv && gv->isConstant())
attr |= LTO_SYMBOL_PERMISSIONS_RODATA;
else
attr |= LTO_SYMBOL_PERMISSIONS_DATA;
}
// set definition part
if (def->hasWeakLinkage() || def->hasLinkOnceLinkage() ||
def->hasLinkerPrivateWeakLinkage())
attr |= LTO_SYMBOL_DEFINITION_WEAK;
else if (def->hasCommonLinkage())
attr |= LTO_SYMBOL_DEFINITION_TENTATIVE;
else
attr |= LTO_SYMBOL_DEFINITION_REGULAR;
// set scope part
if (def->hasHiddenVisibility())
attr |= LTO_SYMBOL_SCOPE_HIDDEN;
else if (def->hasProtectedVisibility())
attr |= LTO_SYMBOL_SCOPE_PROTECTED;
else if (def->hasExternalLinkage() || def->hasWeakLinkage() ||
def->hasLinkOnceLinkage() || def->hasCommonLinkage() ||
def->hasLinkerPrivateWeakLinkage())
attr |= LTO_SYMBOL_SCOPE_DEFAULT;
else if (def->hasLinkOnceODRAutoHideLinkage())
attr |= LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN;
else
attr |= LTO_SYMBOL_SCOPE_INTERNAL;
StringSet::value_type &entry = _defines.GetOrCreateValue(Buffer);
entry.setValue(1);
// fill information structure
NameAndAttributes info;
StringRef Name = entry.getKey();
info.name = Name.data();
assert(info.name[Name.size()] == '\0');
info.attributes = attr;
info.isFunction = isFunction;
info.symbol = def;
// add to table of symbols
_symbols.push_back(info);
}
void GCOVProfiler::insertCounterWriteout(
SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> &CountersBySP) {
FunctionType *WriteoutFTy =
FunctionType::get(Type::getVoidTy(*Ctx), false);
Function *WriteoutF = Function::Create(WriteoutFTy,
GlobalValue::InternalLinkage,
"__llvm_gcov_writeout", M);
WriteoutF->setUnnamedAddr(true);
BasicBlock *BB = BasicBlock::Create(*Ctx, "", WriteoutF);
IRBuilder<> Builder(BB);
Constant *StartFile = getStartFileFunc();
Constant *EmitFunction = getEmitFunctionFunc();
Constant *EmitArcs = getEmitArcsFunc();
Constant *EndFile = getEndFileFunc();
NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
if (CU_Nodes) {
for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
DICompileUnit compile_unit(CU_Nodes->getOperand(i));
std::string FilenameGcda = mangleName(compile_unit, "gcda");
Builder.CreateCall(StartFile,
Builder.CreateGlobalStringPtr(FilenameGcda));
for (SmallVector<std::pair<GlobalVariable *, MDNode *>, 8>::iterator
I = CountersBySP.begin(), E = CountersBySP.end();
I != E; ++I) {
DISubprogram SP(I->second);
intptr_t ident = reinterpret_cast<intptr_t>(I->second);
Builder.CreateCall2(EmitFunction,
ConstantInt::get(Type::getInt32Ty(*Ctx), ident),
Builder.CreateGlobalStringPtr(SP.getName()));
GlobalVariable *GV = I->first;
unsigned Arcs =
cast<ArrayType>(GV->getType()->getElementType())->getNumElements();
Builder.CreateCall2(EmitArcs,
ConstantInt::get(Type::getInt32Ty(*Ctx), Arcs),
Builder.CreateConstGEP2_64(GV, 0, 0));
}
Builder.CreateCall(EndFile);
}
}
Builder.CreateRetVoid();
// Create a small bit of code that registers the "__llvm_gcov_writeout"
// function to be executed at exit.
FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
Function *F = Function::Create(FTy, GlobalValue::InternalLinkage,
"__llvm_gcov_init", M);
F->setUnnamedAddr(true);
F->setLinkage(GlobalValue::InternalLinkage);
F->addFnAttr(Attribute::NoInline);
BB = BasicBlock::Create(*Ctx, "entry", F);
Builder.SetInsertPoint(BB);
FTy = FunctionType::get(Type::getInt32Ty(*Ctx),
PointerType::get(FTy, 0), false);
Function *AtExitFn =
Function::Create(FTy, GlobalValue::ExternalLinkage, "atexit", M);
Builder.CreateCall(AtExitFn, WriteoutF);
Builder.CreateRetVoid();
appendToGlobalCtors(*M, F, 0);
}
std::unique_ptr<StructInfo> StructInfo::getFromGlobalPointer(Module *module, llvm::StringRef name)
{
GlobalVariable *var = module->getGlobalVariable(name, false);
if (!var || !var->getType() || !var->getType()->isPointerTy()) {
assert(false);
llvm::errs() << "StructInfo: Cannot get global variable " << name << ", or it is not a pointer." << '\n';
return nullptr;
}
PointerType *varDeref = dyn_cast<PointerType>(var->getType()->getElementType());
if (!varDeref || !varDeref->getElementType())
{
assert(false);
llvm::errs() << "StructInfo: Pointer not valid." << '\n';
return nullptr;
}
StructType *structType = dyn_cast<StructType>(varDeref->getElementType());
if (!structType) {
assert(false);
llvm::errs() << "StructInfo: Cannot get struct type." << '\n';
return nullptr;
}
NamedMDNode *mdCuNodes = module->getNamedMetadata("llvm.dbg.cu");
if (!mdCuNodes) {
assert(false);
llvm::errs() << "StructInfo: Cannot find metadata." << '\n';
return nullptr;
}
std::shared_ptr<DITypeIdentifierMap> typeIdentifierMap(new DITypeIdentifierMap(generateDITypeIdentifierMap(mdCuNodes)));
DICompositeType *diStructType = nullptr;
for ( unsigned i = 0; i < mdCuNodes->getNumOperands() && !diStructType; ++i )
{
DICompileUnit diCu(mdCuNodes->getOperand(i));
for ( unsigned j = 0; j < diCu.getGlobalVariables().getNumElements(); ++j )
{
DIGlobalVariable diGlobalVar(diCu.getGlobalVariables().getElement(j));
if (diGlobalVar.getName() != name) {
continue;
}
//Go through pointers until we reach a structure
DIType diStructType(diGlobalVar.getType());
while (diStructType.isDerivedType() && !diStructType.isCompositeType()) {
diStructType = std::unique_ptr<DIDerivedType>(new DIDerivedType(diStructType))->getTypeDerivedFrom().resolve(*typeIdentifierMap);
}
if (!diStructType.isCompositeType()) {
llvm::errs() << "StructInfo: Global variable " << name << " does not point to a composite type: " << diStructType.getName() << '\n';
assert(false);
return nullptr;
}
return std::unique_ptr<StructInfo>(new StructInfo(
module,
structType,
new DICompositeType(diStructType),
typeIdentifierMap));
}
}
assert(false);
llvm::errs() << "StructInfo: Did not find global variable " << name << " in debug information." << '\n';
return nullptr;
}
bool StripDeadDebugInfo::runOnModule(Module &M) {
bool Changed = false;
// Debugging infomration is encoded in llvm IR using metadata. This is designed
// such a way that debug info for symbols preserved even if symbols are
// optimized away by the optimizer. This special pass removes debug info for
// such symbols.
// llvm.dbg.gv keeps track of debug info for global variables.
if (NamedMDNode *NMD = M.getNamedMetadata("llvm.dbg.gv")) {
SmallVector<MDNode *, 8> MDs;
for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i)
if (DIGlobalVariable(NMD->getOperand(i)).Verify())
MDs.push_back(NMD->getOperand(i));
else
Changed = true;
NMD->eraseFromParent();
NMD = NULL;
for (SmallVector<MDNode *, 8>::iterator I = MDs.begin(),
E = MDs.end(); I != E; ++I) {
GlobalVariable *GV = DIGlobalVariable(*I).getGlobal();
if (GV && M.getGlobalVariable(GV->getName(), true)) {
if (!NMD)
NMD = M.getOrInsertNamedMetadata("llvm.dbg.gv");
NMD->addOperand(*I);
}
else
Changed = true;
}
}
// llvm.dbg.sp keeps track of debug info for subprograms.
if (NamedMDNode *NMD = M.getNamedMetadata("llvm.dbg.sp")) {
SmallVector<MDNode *, 8> MDs;
for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i)
if (DISubprogram(NMD->getOperand(i)).Verify())
MDs.push_back(NMD->getOperand(i));
else
Changed = true;
NMD->eraseFromParent();
NMD = NULL;
for (SmallVector<MDNode *, 8>::iterator I = MDs.begin(),
E = MDs.end(); I != E; ++I) {
bool FnIsLive = false;
if (Function *F = DISubprogram(*I).getFunction())
if (M.getFunction(F->getName()))
FnIsLive = true;
if (FnIsLive) {
if (!NMD)
NMD = M.getOrInsertNamedMetadata("llvm.dbg.sp");
NMD->addOperand(*I);
} else {
// Remove llvm.dbg.lv.fnname named mdnode which may have been used
// to hold debug info for dead function's local variables.
StringRef FName = DISubprogram(*I).getLinkageName();
if (FName.empty())
FName = DISubprogram(*I).getName();
if (NamedMDNode *LVNMD =
M.getNamedMetadata(Twine("llvm.dbg.lv.",
getRealLinkageName(FName))))
LVNMD->eraseFromParent();
}
}
}
return Changed;
}
bool OptimalEdgeProfiler::runOnModule(Module &M) {
Function *Main = M.getFunction("main");
if (Main == 0) {
M.getContext().emitWarning("cannot insert edge profiling into a module"
" with no main function");
return false; // No main, no instrumentation!
}
// NumEdges counts all the edges that may be instrumented. Later on its
// decided which edges to actually instrument, to achieve optimal profiling.
// For the entry block a virtual edge (0,entry) is reserved, for each block
// with no successors an edge (BB,0) is reserved. These edges are necessary
// to calculate a truly optimal maximum spanning tree and thus an optimal
// instrumentation.
unsigned NumEdges = 0;
for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
if (F->isDeclaration()) continue;
// Reserve space for (0,entry) edge.
++NumEdges;
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
// Keep track of which blocks need to be instrumented. We don't want to
// instrument blocks that are added as the result of breaking critical
// edges!
if (BB->getTerminator()->getNumSuccessors() == 0) {
// Reserve space for (BB,0) edge.
++NumEdges;
} else {
NumEdges += BB->getTerminator()->getNumSuccessors();
}
}
}
// In the profiling output a counter for each edge is reserved, but only few
// are used. This is done to be able to read back in the profile without
// calulating the maximum spanning tree again, instead each edge counter that
// is not used is initialised with -1 to signal that this edge counter has to
// be calculated from other edge counters on reading the profile info back
// in.
Type *Int32 = Type::getInt32Ty(M.getContext());
ArrayType *ATy = ArrayType::get(Int32, NumEdges);
GlobalVariable *Counters =
new GlobalVariable(M, ATy, false, GlobalValue::InternalLinkage,
Constant::getNullValue(ATy), "OptEdgeProfCounters");
NumEdgesInserted = 0;
std::vector<Constant*> Initializer(NumEdges);
Constant *Zero = ConstantInt::get(Int32, 0);
Constant *Uncounted = ConstantInt::get(Int32, ProfileInfoLoader::Uncounted);
// Instrument all of the edges not in MST...
unsigned i = 0;
for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
if (F->isDeclaration()) continue;
DEBUG(dbgs() << "Working on " << F->getName() << "\n");
// Calculate a Maximum Spanning Tree with the edge weights determined by
// ProfileEstimator. ProfileEstimator also assign weights to the virtual
// edges (0,entry) and (BB,0) (for blocks with no successors) and this
// edges also participate in the maximum spanning tree calculation.
// The third parameter of MaximumSpanningTree() has the effect that not the
// actual MST is returned but the edges _not_ in the MST.
ProfileInfo::EdgeWeights ECs =
getAnalysis<ProfileInfo>(*F).getEdgeWeights(F);
std::vector<ProfileInfo::EdgeWeight> EdgeVector(ECs.begin(), ECs.end());
MaximumSpanningTree<BasicBlock> MST(EdgeVector);
std::stable_sort(MST.begin(), MST.end());
// Check if (0,entry) not in the MST. If not, instrument edge
// (IncrementCounterInBlock()) and set the counter initially to zero, if
// the edge is in the MST the counter is initialised to -1.
BasicBlock *entry = &(F->getEntryBlock());
ProfileInfo::Edge edge = ProfileInfo::getEdge(0, entry);
if (!std::binary_search(MST.begin(), MST.end(), edge)) {
printEdgeCounter(edge, entry, i);
IncrementCounterInBlock(entry, i, Counters); ++NumEdgesInserted;
Initializer[i++] = (Zero);
} else{
Initializer[i++] = (Uncounted);
}
// InsertedBlocks contains all blocks that were inserted for splitting an
// edge, this blocks do not have to be instrumented.
DenseSet<BasicBlock*> InsertedBlocks;
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
// Check if block was not inserted and thus does not have to be
// instrumented.
if (InsertedBlocks.count(BB)) continue;
// Okay, we have to add a counter of each outgoing edge not in MST. If
// the outgoing edge is not critical don't split it, just insert the
// counter in the source or destination of the edge. Also, if the block
// has no successors, the virtual edge (BB,0) is processed.
TerminatorInst *TI = BB->getTerminator();
if (TI->getNumSuccessors() == 0) {
ProfileInfo::Edge edge = ProfileInfo::getEdge(BB, 0);
if (!std::binary_search(MST.begin(), MST.end(), edge)) {
printEdgeCounter(edge, BB, i);
IncrementCounterInBlock(BB, i, Counters); ++NumEdgesInserted;
Initializer[i++] = (Zero);
} else{
Initializer[i++] = (Uncounted);
}
}
for (unsigned s = 0, e = TI->getNumSuccessors(); s != e; ++s) {
BasicBlock *Succ = TI->getSuccessor(s);
ProfileInfo::Edge edge = ProfileInfo::getEdge(BB,Succ);
if (!std::binary_search(MST.begin(), MST.end(), edge)) {
// If the edge is critical, split it.
bool wasInserted = SplitCriticalEdge(TI, s, this);
Succ = TI->getSuccessor(s);
if (wasInserted)
InsertedBlocks.insert(Succ);
// Okay, we are guaranteed that the edge is no longer critical. If
// we only have a single successor, insert the counter in this block,
// otherwise insert it in the successor block.
if (TI->getNumSuccessors() == 1) {
// Insert counter at the start of the block
printEdgeCounter(edge, BB, i);
IncrementCounterInBlock(BB, i, Counters); ++NumEdgesInserted;
} else {
// Insert counter at the start of the block
printEdgeCounter(edge, Succ, i);
IncrementCounterInBlock(Succ, i, Counters); ++NumEdgesInserted;
}
Initializer[i++] = (Zero);
} else {
Initializer[i++] = (Uncounted);
}
}
}
}
// Check if the number of edges counted at first was the number of edges we
// considered for instrumentation.
assert(i == NumEdges && "the number of edges in counting array is wrong");
// Assign the now completely defined initialiser to the array.
Constant *init = ConstantArray::get(ATy, Initializer);
Counters->setInitializer(init);
// Add the initialization call to main.
InsertProfilingInitCall(Main, "llvm_start_opt_edge_profiling", Counters);
return true;
}
bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
const BitVector &GlobalSet, Module &M, bool isConst,
unsigned AddrSpace) const {
Type *Int32Ty = Type::getInt32Ty(M.getContext());
auto &DL = M.getDataLayout();
assert(Globals.size() > 1);
DEBUG(dbgs() << " Trying to merge set, starts with #"
<< GlobalSet.find_first() << "\n");
ssize_t i = GlobalSet.find_first();
while (i != -1) {
ssize_t j = 0;
uint64_t MergedSize = 0;
std::vector<Type*> Tys;
std::vector<Constant*> Inits;
bool HasExternal = false;
GlobalVariable *TheFirstExternal = 0;
for (j = i; j != -1; j = GlobalSet.find_next(j)) {
Type *Ty = Globals[j]->getType()->getElementType();
MergedSize += DL.getTypeAllocSize(Ty);
if (MergedSize > MaxOffset) {
break;
}
Tys.push_back(Ty);
Inits.push_back(Globals[j]->getInitializer());
if (Globals[j]->hasExternalLinkage() && !HasExternal) {
HasExternal = true;
TheFirstExternal = Globals[j];
}
}
// If merged variables doesn't have external linkage, we needn't to expose
// the symbol after merging.
GlobalValue::LinkageTypes Linkage = HasExternal
? GlobalValue::ExternalLinkage
: GlobalValue::InternalLinkage;
StructType *MergedTy = StructType::get(M.getContext(), Tys);
Constant *MergedInit = ConstantStruct::get(MergedTy, Inits);
// If merged variables have external linkage, we use symbol name of the
// first variable merged as the suffix of global symbol name. This would
// be able to avoid the link-time naming conflict for globalm symbols.
GlobalVariable *MergedGV = new GlobalVariable(
M, MergedTy, isConst, Linkage, MergedInit,
HasExternal ? "_MergedGlobals_" + TheFirstExternal->getName()
: "_MergedGlobals",
nullptr, GlobalVariable::NotThreadLocal, AddrSpace);
for (ssize_t k = i, idx = 0; k != j; k = GlobalSet.find_next(k)) {
GlobalValue::LinkageTypes Linkage = Globals[k]->getLinkage();
std::string Name = Globals[k]->getName();
Constant *Idx[2] = {
ConstantInt::get(Int32Ty, 0),
ConstantInt::get(Int32Ty, idx++)
};
Constant *GEP =
ConstantExpr::getInBoundsGetElementPtr(MergedTy, MergedGV, Idx);
Globals[k]->replaceAllUsesWith(GEP);
Globals[k]->eraseFromParent();
if (Linkage != GlobalValue::InternalLinkage) {
// Generate a new alias...
auto *PTy = cast<PointerType>(GEP->getType());
GlobalAlias::create(PTy, Linkage, Name, GEP, &M);
}
NumMerged++;
}
i = j;
}
return true;
}
/// SplitFunctionsOutOfModule - Given a module and a list of functions in the
/// module, split the functions OUT of the specified module, and place them in
/// the new module.
Module *
llvm::SplitFunctionsOutOfModule(Module *M,
const std::vector<Function*> &F,
ValueToValueMapTy &VMap) {
// Make sure functions & globals are all external so that linkage
// between the two modules will work.
for (Module::iterator I = M->begin(), E = M->end(); I != E; ++I)
I->setLinkage(GlobalValue::ExternalLinkage);
for (Module::global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
if (I->hasName() && I->getName()[0] == '\01')
I->setName(I->getName().substr(1));
I->setLinkage(GlobalValue::ExternalLinkage);
}
ValueToValueMapTy NewVMap;
Module *New = CloneModule(M, NewVMap);
// Remove the Test functions from the Safe module
std::set<Function *> TestFunctions;
for (unsigned i = 0, e = F.size(); i != e; ++i) {
Function *TNOF = cast<Function>(VMap[F[i]]);
DEBUG(errs() << "Removing function ");
DEBUG(WriteAsOperand(errs(), TNOF, false));
DEBUG(errs() << "\n");
TestFunctions.insert(cast<Function>(NewVMap[TNOF]));
DeleteFunctionBody(TNOF); // Function is now external in this module!
}
// Remove the Safe functions from the Test module
for (Module::iterator I = New->begin(), E = New->end(); I != E; ++I)
if (!TestFunctions.count(I))
DeleteFunctionBody(I);
// Try to split the global initializers evenly
for (Module::global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(NewVMap[I]);
if (Function *TestFn = globalInitUsesExternalBA(I)) {
if (Function *SafeFn = globalInitUsesExternalBA(GV)) {
errs() << "*** Error: when reducing functions, encountered "
"the global '";
WriteAsOperand(errs(), GV, false);
errs() << "' with an initializer that references blockaddresses "
"from safe function '" << SafeFn->getName()
<< "' and from test function '" << TestFn->getName() << "'.\n";
exit(1);
}
I->setInitializer(0); // Delete the initializer to make it external
} else {
// If we keep it in the safe module, then delete it in the test module
GV->setInitializer(0);
}
}
// Make sure that there is a global ctor/dtor array in both halves of the
// module if they both have static ctor/dtor functions.
SplitStaticCtorDtor("llvm.global_ctors", M, New, NewVMap);
SplitStaticCtorDtor("llvm.global_dtors", M, New, NewVMap);
return New;
}
std::unique_ptr<Module> llvm::CloneModule(
const Module *M, ValueToValueMapTy &VMap,
std::function<bool(const GlobalValue *)> ShouldCloneDefinition) {
// First off, we need to create the new module.
std::unique_ptr<Module> New =
llvm::make_unique<Module>(M->getModuleIdentifier(), M->getContext());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the VMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(*New,
I->getValueType(),
I->isConstant(), I->getLinkage(),
(Constant*) nullptr, I->getName(),
(GlobalVariable*) nullptr,
I->getThreadLocalMode(),
I->getType()->getAddressSpace());
GV->copyAttributesFrom(&*I);
VMap[&*I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getValueType()),
I->getLinkage(), I->getName(), New.get());
NF->copyAttributesFrom(&*I);
VMap[&*I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
if (!ShouldCloneDefinition(&*I)) {
// An alias cannot act as an external reference, so we need to create
// either a function or a global variable depending on the value type.
// FIXME: Once pointee types are gone we can probably pick one or the
// other.
GlobalValue *GV;
if (I->getValueType()->isFunctionTy())
GV = Function::Create(cast<FunctionType>(I->getValueType()),
GlobalValue::ExternalLinkage, I->getName(),
New.get());
else
GV = new GlobalVariable(
*New, I->getValueType(), false, GlobalValue::ExternalLinkage,
(Constant *)nullptr, I->getName(), (GlobalVariable *)nullptr,
I->getThreadLocalMode(), I->getType()->getAddressSpace());
VMap[&*I] = GV;
// We do not copy attributes (mainly because copying between different
// kinds of globals is forbidden), but this is generally not required for
// correctness.
continue;
}
auto *GA = GlobalAlias::create(I->getValueType(),
I->getType()->getPointerAddressSpace(),
I->getLinkage(), I->getName(), New.get());
GA->copyAttributesFrom(&*I);
VMap[&*I] = GA;
}
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(VMap[&*I]);
if (!ShouldCloneDefinition(&*I)) {
// Skip after setting the correct linkage for an external reference.
GV->setLinkage(GlobalValue::ExternalLinkage);
continue;
}
if (I->hasInitializer())
GV->setInitializer(MapValue(I->getInitializer(), VMap));
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(VMap[&*I]);
if (!ShouldCloneDefinition(&*I)) {
// Skip after setting the correct linkage for an external reference.
F->setLinkage(GlobalValue::ExternalLinkage);
// Personality function is not valid on a declaration.
F->setPersonalityFn(nullptr);
continue;
}
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
VMap[&*J] = &*DestI++;
}
SmallVector<ReturnInst*, 8> Returns; // Ignore returns cloned.
CloneFunctionInto(F, &*I, VMap, /*ModuleLevelChanges=*/true, Returns);
}
if (I->hasPersonalityFn())
F->setPersonalityFn(MapValue(I->getPersonalityFn(), VMap));
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
// We already dealt with undefined aliases above.
if (!ShouldCloneDefinition(&*I))
continue;
GlobalAlias *GA = cast<GlobalAlias>(VMap[&*I]);
if (const Constant *C = I->getAliasee())
GA->setAliasee(MapValue(C, VMap));
}
// And named metadata....
for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
E = M->named_metadata_end(); I != E; ++I) {
const NamedMDNode &NMD = *I;
NamedMDNode *NewNMD = New->getOrInsertNamedMetadata(NMD.getName());
for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
NewNMD->addOperand(MapMetadata(NMD.getOperand(i), VMap));
}
return New;
}
Function *GCOVProfiler::insertCounterWriteout(
ArrayRef<std::pair<GlobalVariable *, MDNode *> > CountersBySP) {
FunctionType *WriteoutFTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
Function *WriteoutF = M->getFunction("__llvm_gcov_writeout");
if (!WriteoutF)
WriteoutF = Function::Create(WriteoutFTy, GlobalValue::InternalLinkage,
"__llvm_gcov_writeout", M);
WriteoutF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
WriteoutF->addFnAttr(Attribute::NoInline);
if (Options.NoRedZone)
WriteoutF->addFnAttr(Attribute::NoRedZone);
BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", WriteoutF);
IRBuilder<> Builder(BB);
Constant *StartFile = getStartFileFunc();
Constant *EmitFunction = getEmitFunctionFunc();
Constant *EmitArcs = getEmitArcsFunc();
Constant *SummaryInfo = getSummaryInfoFunc();
Constant *EndFile = getEndFileFunc();
NamedMDNode *CUNodes = M->getNamedMetadata("llvm.dbg.cu");
if (!CUNodes) {
Builder.CreateRetVoid();
return WriteoutF;
}
// Collect the relevant data into a large constant data structure that we can
// walk to write out everything.
StructType *StartFileCallArgsTy = StructType::create(
{Builder.getInt8PtrTy(), Builder.getInt8PtrTy(), Builder.getInt32Ty()});
StructType *EmitFunctionCallArgsTy = StructType::create(
{Builder.getInt32Ty(), Builder.getInt8PtrTy(), Builder.getInt32Ty(),
Builder.getInt8Ty(), Builder.getInt32Ty()});
StructType *EmitArcsCallArgsTy = StructType::create(
{Builder.getInt32Ty(), Builder.getInt64Ty()->getPointerTo()});
StructType *FileInfoTy =
StructType::create({StartFileCallArgsTy, Builder.getInt32Ty(),
EmitFunctionCallArgsTy->getPointerTo(),
EmitArcsCallArgsTy->getPointerTo()});
Constant *Zero32 = Builder.getInt32(0);
// Build an explicit array of two zeros for use in ConstantExpr GEP building.
Constant *TwoZero32s[] = {Zero32, Zero32};
SmallVector<Constant *, 8> FileInfos;
for (int i : llvm::seq<int>(0, CUNodes->getNumOperands())) {
auto *CU = cast<DICompileUnit>(CUNodes->getOperand(i));
// Skip module skeleton (and module) CUs.
if (CU->getDWOId())
continue;
std::string FilenameGcda = mangleName(CU, GCovFileType::GCDA);
uint32_t CfgChecksum = FileChecksums.empty() ? 0 : FileChecksums[i];
auto *StartFileCallArgs = ConstantStruct::get(
StartFileCallArgsTy, {Builder.CreateGlobalStringPtr(FilenameGcda),
Builder.CreateGlobalStringPtr(ReversedVersion),
Builder.getInt32(CfgChecksum)});
SmallVector<Constant *, 8> EmitFunctionCallArgsArray;
SmallVector<Constant *, 8> EmitArcsCallArgsArray;
for (int j : llvm::seq<int>(0, CountersBySP.size())) {
auto *SP = cast_or_null<DISubprogram>(CountersBySP[j].second);
uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[j]->getFuncChecksum();
EmitFunctionCallArgsArray.push_back(ConstantStruct::get(
EmitFunctionCallArgsTy,
{Builder.getInt32(j),
Options.FunctionNamesInData
? Builder.CreateGlobalStringPtr(getFunctionName(SP))
: Constant::getNullValue(Builder.getInt8PtrTy()),
Builder.getInt32(FuncChecksum),
Builder.getInt8(Options.UseCfgChecksum),
Builder.getInt32(CfgChecksum)}));
GlobalVariable *GV = CountersBySP[j].first;
unsigned Arcs = cast<ArrayType>(GV->getValueType())->getNumElements();
EmitArcsCallArgsArray.push_back(ConstantStruct::get(
EmitArcsCallArgsTy,
{Builder.getInt32(Arcs), ConstantExpr::getInBoundsGetElementPtr(
GV->getValueType(), GV, TwoZero32s)}));
}
// Create global arrays for the two emit calls.
int CountersSize = CountersBySP.size();
assert(CountersSize == (int)EmitFunctionCallArgsArray.size() &&
"Mismatched array size!");
assert(CountersSize == (int)EmitArcsCallArgsArray.size() &&
"Mismatched array size!");
auto *EmitFunctionCallArgsArrayTy =
ArrayType::get(EmitFunctionCallArgsTy, CountersSize);
auto *EmitFunctionCallArgsArrayGV = new GlobalVariable(
*M, EmitFunctionCallArgsArrayTy, /*isConstant*/ true,
GlobalValue::InternalLinkage,
ConstantArray::get(EmitFunctionCallArgsArrayTy,
EmitFunctionCallArgsArray),
Twine("__llvm_internal_gcov_emit_function_args.") + Twine(i));
auto *EmitArcsCallArgsArrayTy =
ArrayType::get(EmitArcsCallArgsTy, CountersSize);
EmitFunctionCallArgsArrayGV->setUnnamedAddr(
GlobalValue::UnnamedAddr::Global);
auto *EmitArcsCallArgsArrayGV = new GlobalVariable(
*M, EmitArcsCallArgsArrayTy, /*isConstant*/ true,
GlobalValue::InternalLinkage,
ConstantArray::get(EmitArcsCallArgsArrayTy, EmitArcsCallArgsArray),
Twine("__llvm_internal_gcov_emit_arcs_args.") + Twine(i));
EmitArcsCallArgsArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
FileInfos.push_back(ConstantStruct::get(
FileInfoTy,
{StartFileCallArgs, Builder.getInt32(CountersSize),
ConstantExpr::getInBoundsGetElementPtr(EmitFunctionCallArgsArrayTy,
EmitFunctionCallArgsArrayGV,
TwoZero32s),
ConstantExpr::getInBoundsGetElementPtr(
EmitArcsCallArgsArrayTy, EmitArcsCallArgsArrayGV, TwoZero32s)}));
}
// If we didn't find anything to actually emit, bail on out.
if (FileInfos.empty()) {
Builder.CreateRetVoid();
return WriteoutF;
}
// To simplify code, we cap the number of file infos we write out to fit
// easily in a 32-bit signed integer. This gives consistent behavior between
// 32-bit and 64-bit systems without requiring (potentially very slow) 64-bit
// operations on 32-bit systems. It also seems unreasonable to try to handle
// more than 2 billion files.
if ((int64_t)FileInfos.size() > (int64_t)INT_MAX)
FileInfos.resize(INT_MAX);
// Create a global for the entire data structure so we can walk it more
// easily.
auto *FileInfoArrayTy = ArrayType::get(FileInfoTy, FileInfos.size());
auto *FileInfoArrayGV = new GlobalVariable(
*M, FileInfoArrayTy, /*isConstant*/ true, GlobalValue::InternalLinkage,
ConstantArray::get(FileInfoArrayTy, FileInfos),
"__llvm_internal_gcov_emit_file_info");
FileInfoArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
// Create the CFG for walking this data structure.
auto *FileLoopHeader =
BasicBlock::Create(*Ctx, "file.loop.header", WriteoutF);
auto *CounterLoopHeader =
BasicBlock::Create(*Ctx, "counter.loop.header", WriteoutF);
auto *FileLoopLatch = BasicBlock::Create(*Ctx, "file.loop.latch", WriteoutF);
auto *ExitBB = BasicBlock::Create(*Ctx, "exit", WriteoutF);
// We always have at least one file, so just branch to the header.
Builder.CreateBr(FileLoopHeader);
// The index into the files structure is our loop induction variable.
Builder.SetInsertPoint(FileLoopHeader);
PHINode *IV =
Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
IV->addIncoming(Builder.getInt32(0), BB);
auto *FileInfoPtr =
Builder.CreateInBoundsGEP(FileInfoArrayGV, {Builder.getInt32(0), IV});
auto *StartFileCallArgsPtr = Builder.CreateStructGEP(FileInfoPtr, 0);
Builder.CreateCall(
StartFile,
{Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 0)),
Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 1)),
Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 2))});
auto *NumCounters =
Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 1));
auto *EmitFunctionCallArgsArray =
Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 2));
auto *EmitArcsCallArgsArray =
Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 3));
auto *EnterCounterLoopCond =
Builder.CreateICmpSLT(Builder.getInt32(0), NumCounters);
Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
Builder.SetInsertPoint(CounterLoopHeader);
auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
JV->addIncoming(Builder.getInt32(0), FileLoopHeader);
auto *EmitFunctionCallArgsPtr =
Builder.CreateInBoundsGEP(EmitFunctionCallArgsArray, {JV});
Builder.CreateCall(
EmitFunction,
{Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 0)),
Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 1)),
Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 2)),
Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 3)),
Builder.CreateLoad(
Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 4))});
auto *EmitArcsCallArgsPtr =
Builder.CreateInBoundsGEP(EmitArcsCallArgsArray, {JV});
Builder.CreateCall(
EmitArcs,
{Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 0)),
Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 1))});
auto *NextJV = Builder.CreateAdd(JV, Builder.getInt32(1));
auto *CounterLoopCond = Builder.CreateICmpSLT(NextJV, NumCounters);
Builder.CreateCondBr(CounterLoopCond, CounterLoopHeader, FileLoopLatch);
JV->addIncoming(NextJV, CounterLoopHeader);
Builder.SetInsertPoint(FileLoopLatch);
Builder.CreateCall(SummaryInfo, {});
Builder.CreateCall(EndFile, {});
auto *NextIV = Builder.CreateAdd(IV, Builder.getInt32(1));
auto *FileLoopCond =
Builder.CreateICmpSLT(NextIV, Builder.getInt32(FileInfos.size()));
Builder.CreateCondBr(FileLoopCond, FileLoopHeader, ExitBB);
IV->addIncoming(NextIV, FileLoopLatch);
Builder.SetInsertPoint(ExitBB);
Builder.CreateRetVoid();
return WriteoutF;
}
bool BlackList::isIn(const GlobalVariable &G) {
return isIn(*G.getParent()) || inSection("global", G.getName());
}
//
// Method: postOrderInline()
//
// Description:
// This methods does a post order traversal of the call graph and performs
// bottom-up inlining of the DSGraphs.
//
void
BUDataStructures::postOrderInline (Module & M) {
// Variables used for Tarjan SCC-finding algorithm. These are passed into
// the recursive function used to find SCCs.
std::vector<const Function*> Stack;
std::map<const Function*, unsigned> ValMap;
unsigned NextID = 1;
// Do post order traversal on the global ctors. Use this information to update
// the globals graph.
const char *Name = "llvm.global_ctors";
GlobalVariable *GV = M.getNamedGlobal(Name);
if (GV && !(GV->isDeclaration()) && !(GV->hasLocalLinkage())) {
// Should be an array of '{ int, void ()* }' structs. The first value is
// the init priority, which we ignore.
ConstantArray *InitList = dyn_cast<ConstantArray>(GV->getInitializer());
if (InitList) {
for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i)
if (ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i))) {
if (CS->getNumOperands() != 2)
break; // Not array of 2-element structs.
Constant *FP = CS->getOperand(1);
if (FP->isNullValue())
break; // Found a null terminator, exit.
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(FP))
if (CE->isCast())
FP = CE->getOperand(0);
Function *F = dyn_cast<Function>(FP);
if (F && !F->isDeclaration() && !ValMap.count(F)) {
calculateGraphs(F, Stack, NextID, ValMap);
CloneAuxIntoGlobal(getDSGraph(*F));
}
}
GlobalsGraph->removeTriviallyDeadNodes();
GlobalsGraph->maskIncompleteMarkers();
// Mark external globals incomplete.
GlobalsGraph->markIncompleteNodes(DSGraph::IgnoreGlobals);
GlobalsGraph->computeExternalFlags(DSGraph::DontMarkFormalsExternal);
GlobalsGraph->computeIntPtrFlags();
//
// Create equivalence classes for aliasing globals so that we only need to
// record one global per DSNode.
//
formGlobalECs();
// propogte information calculated
// from the globals graph to the other graphs.
for (Module::iterator F = M.begin(); F != M.end(); ++F) {
if (!(F->isDeclaration())){
DSGraph *Graph = getDSGraph(*F);
cloneGlobalsInto(Graph, DSGraph::DontCloneCallNodes |
DSGraph::DontCloneAuxCallNodes);
Graph->buildCallGraph(callgraph, GlobalFunctionList, filterCallees);
Graph->maskIncompleteMarkers();
Graph->markIncompleteNodes(DSGraph::MarkFormalArgs |
DSGraph::IgnoreGlobals);
Graph->computeExternalFlags(DSGraph::DontMarkFormalsExternal);
Graph->computeIntPtrFlags();
}
}
}
}
//
// Start the post order traversal with the main() function. If there is no
// main() function, don't worry; we'll have a separate traversal for inlining
// graphs for functions not reachable from main().
//
Function *MainFunc = M.getFunction ("main");
if (MainFunc && !MainFunc->isDeclaration()) {
calculateGraphs(MainFunc, Stack, NextID, ValMap);
CloneAuxIntoGlobal(getDSGraph(*MainFunc));
}
//
// Calculate the graphs for any functions that are unreachable from main...
//
for (Function &F : M)
if (!F.isDeclaration() && !ValMap.count(&F)) {
if (MainFunc)
DEBUG(errs() << debugname << ": Function unreachable from main: "
<< F.getName() << "\n");
calculateGraphs(&F, Stack, NextID, ValMap); // Calculate all graphs.
CloneAuxIntoGlobal(getDSGraph(F));
// Mark this graph as processed. Do this by finding all functions
// in the graph that map to it, and mark them visited.
// Note that this really should be handled neatly by calculateGraphs
// itself, not here. However this catches the worst offenders.
DSGraph *G = getDSGraph(F);
for(DSGraph::retnodes_iterator RI = G->retnodes_begin(),
RE = G->retnodes_end(); RI != RE; ++RI) {
if (getDSGraph(*RI->first) == G) {
if (!ValMap.count(RI->first))
ValMap[RI->first] = ~0U;
else
assert(ValMap[RI->first] == ~0U);
}
}
}
return;
}