void WorklessInstrument::SetupGlobals(Module * pModule)
{
assert(pModule->getGlobalVariable("SAMPLE_RATE")==NULL);
this->SAMPLE_RATE = new GlobalVariable(*pModule, this->IntType, false, GlobalValue::CommonLinkage, 0, "SAMPLE_RATE");
this->SAMPLE_RATE->setAlignment(4);
this->SAMPLE_RATE->setInitializer(this->ConstantInt0);
assert(pModule->getGlobalVariable("PC_SAMPLE_RATE")==NULL);
this->PC_SAMPLE_RATE = new GlobalVariable(*pModule, this->CharStarType, false, GlobalValue::CommonLinkage, 0, "PC_SAMPLE_RATE");
this->PC_SAMPLE_RATE->setAlignment(8);
this->PC_SAMPLE_RATE->setInitializer(this->ConstantNULL);
assert(pModule->getGlobalVariable("numGlobalCounter")==NULL);
this->numGlobalCounter = new GlobalVariable( *pModule , this->LongType, false, GlobalValue::ExternalLinkage, 0, "numGlobalCounter");
this->numGlobalCounter->setAlignment(8);
this->numGlobalCounter->setInitializer(this->ConstantLong0);
/*
assert(pModule->getGlobalVariable("numInstances")==NULL);
this->numInstances = new GlobalVariable(*pModule, this->LongType, false, GlobalVariable::ExternalLinkage, 0, "numInstances");
this->numInstances->setAlignment(8);
this->numInstances->setInitializer(this->ConstantLong0);
*/
assert(pModule->getGlobalVariable("CURRENT_SAMPLE") == NULL);
this->CURRENT_SAMPLE = new GlobalVariable(*pModule, this->LongType, false, GlobalValue::ExternalLinkage, 0, "CURRENT_SAMPLE");
this->CURRENT_SAMPLE->setAlignment(8);
this->CURRENT_SAMPLE->setInitializer(this->ConstantLong0);
//"SAMPLE_RATE" string
ArrayType* ArrayTy12 = ArrayType::get(IntegerType::get(pModule->getContext(), 8), 12);
GlobalVariable * pArrayStr = new GlobalVariable(*pModule, ArrayTy12, true, GlobalValue::PrivateLinkage, 0, "");
pArrayStr->setAlignment(1);
Constant * ConstArray = ConstantDataArray::getString(pModule->getContext(), "SAMPLE_RATE", true);
vector<Constant *> vecIndex;
vecIndex.push_back(this->ConstantInt0);
vecIndex.push_back(this->ConstantInt0);
this->SAMPLE_RATE_ptr = ConstantExpr::getGetElementPtr(pArrayStr, vecIndex);
pArrayStr->setInitializer(ConstArray);
//""
ArrayType * ArrayTy17 = ArrayType::get(IntegerType::get(pModule->getContext(), 8), 17);
pArrayStr = new GlobalVariable(*pModule, ArrayTy17, true, GlobalValue::PrivateLinkage, 0, "");
pArrayStr->setAlignment(1);
ConstArray = ConstantDataArray::getString(pModule->getContext(), "SAMPLE_RATE: %d\x0A", true);
vecIndex.clear();
vecIndex.push_back(this->ConstantInt0);
vecIndex.push_back(this->ConstantInt0);
this->Output_Format_String = ConstantExpr::getGetElementPtr(pArrayStr, vecIndex);
pArrayStr->setInitializer(ConstArray);
}
void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
CoverageData->setSection(getCoverageSection());
CoverageData->setAlignment(8);
Constant *Init = CoverageData->getInitializer();
// We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
// for some C. If not, the frontend's given us something broken.
assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
"invalid function list in coverage map");
ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
Constant *Record = Records->getOperand(I);
Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
assert(isa<GlobalVariable>(V) && "Missing reference to function name");
GlobalVariable *Name = cast<GlobalVariable>(V);
// If we have region counters for this name, we've already handled it.
auto It = RegionCounters.find(Name);
if (It != RegionCounters.end())
continue;
// Move the name variable to the right section.
Name->setSection(getNameSection());
Name->setAlignment(1);
}
}
// On every indirect call we call a run-time function
// __sanitizer_cov_indir_call* with two parameters:
// - callee address,
// - global cache array that contains CacheSize pointers (zero-initialized).
// The cache is used to speed up recording the caller-callee pairs.
// The address of the caller is passed implicitly via caller PC.
// CacheSize is encoded in the name of the run-time function.
void SanitizerCoverageModule::InjectCoverageForIndirectCalls(
Function &F, ArrayRef<Instruction *> IndirCalls) {
if (IndirCalls.empty())
return;
const int CacheSize = 16;
const int CacheAlignment = 64; // Align for better performance.
Type *Ty = ArrayType::get(IntptrTy, CacheSize);
for (auto I : IndirCalls) {
IRBuilder<> IRB(I);
CallSite CS(I);
Value *Callee = CS.getCalledValue();
if (isa<InlineAsm>(Callee))
continue;
GlobalVariable *CalleeCache = new GlobalVariable(
*F.getParent(), Ty, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Ty), "__sancov_gen_callee_cache");
CalleeCache->setAlignment(CacheAlignment);
if (Options.TracePC || Options.TracePCGuard)
IRB.CreateCall(SanCovTracePCIndir,
IRB.CreatePointerCast(Callee, IntptrTy));
else
IRB.CreateCall(SanCovIndirCallFunction,
{IRB.CreatePointerCast(Callee, IntptrTy),
IRB.CreatePointerCast(CalleeCache, IntptrTy)});
}
}
static bool addTableDataSection(NativeModulePtr natMod,
Module *M, VA &newVA, const T& table)
{
list<DataSection> &globaldata = natMod->getData();
list<DataSection>::const_iterator git = globaldata.begin();
// ensure we make this the last data section
newVA = 0;
while( git != globaldata.end() ) {
const DataSection &dt = *git;
uint64_t extent = dt.getBase() + dt.getSize();
if(newVA < extent) {
newVA = extent;
}
git++;
}
// skip a few
newVA += 4;
// create a new data section from the table
DataSection *ds = tableToDataSection(newVA, table);
// add to global data section list
globaldata.push_back(*ds);
// create the GlobalVariable
string bufferName = "data_0x" + to_string<VA>(newVA, hex);
StructType *st_opaque = StructType::create(M->getContext());
GlobalVariable *gv = new GlobalVariable(*M,
st_opaque,
true,
GlobalVariable::InternalLinkage,
NULL,
bufferName);
vector<Type*> data_section_types;
vector<Constant*> secContents;
dataSectionToTypesContents(globaldata,
*ds,
M,
secContents,
data_section_types,
false);
st_opaque->setBody(data_section_types, true);
Constant *cst = ConstantStruct::get(st_opaque, secContents);
gv->setAlignment(4);
gv->setInitializer(cst);
return true;
}
void Variables::changeGlobal(Change* change, Module &module) {
GlobalValue* oldTarget = dyn_cast<GlobalValue>(change->getValue());
Type* oldType = oldTarget->getType()->getElementType();
Type* newType = change->getType()[0];
errs() << "Changing the precision of variable \"" << oldTarget->getName() << "\" from " << *oldType << " to " << *newType << ".\n";
if (diffTypes(oldType, newType)) {
Constant *initializer;
GlobalVariable* newTarget;
if (PointerType *newPointerType = dyn_cast<PointerType>(newType)) {
initializer = ConstantPointerNull::get(newPointerType);
newTarget = new GlobalVariable(module, newType, false, GlobalValue::CommonLinkage, initializer, "");
}
else if (ArrayType * atype = dyn_cast<ArrayType>(newType)) {
// preparing initializer
Type *temp = Type::getFloatTy(module.getContext());
vector<Constant*> operands;
operands.push_back(ConstantFP::get(temp, 0));
ArrayRef<Constant*> *arrayRef = new ArrayRef<Constant*>(operands);
initializer = ConstantArray::get(atype, *arrayRef);
newTarget = new GlobalVariable(module, newType, false, GlobalValue::CommonLinkage, initializer, "");
}
else {
initializer = ConstantFP::get(newType, 0);
newTarget = new GlobalVariable(module, newType, false, GlobalValue::CommonLinkage, initializer, "");
}
/*
GlobalVariable* newTarget = new GlobalVariable(module, newType, false, GlobalValue::CommonLinkage, initializer, "");
*/
unsigned alignment = getAlignment(newType);
newTarget->setAlignment(alignment);
newTarget->takeName(oldTarget);
// iterating through instructions using old AllocaInst
Value::use_iterator it = oldTarget->use_begin();
for(; it != oldTarget->use_end(); it++) {
Transformer::transform(it, newTarget, oldTarget, newType, oldType, alignment);
}
//oldTarget->eraseFromParent();
}
else {
errs() << "No changes required.\n";
}
return;
}
/// Loop through the global variables in the src module and merge them into the
/// dest module.
GlobalVariable *IRLinker::copyGlobalVariableProto(const GlobalVariable *SGVar) {
// No linking to be performed or linking from the source: simply create an
// identical version of the symbol over in the dest module... the
// initializer will be filled in later by LinkGlobalInits.
GlobalVariable *NewDGV =
new GlobalVariable(DstM, TypeMap.get(SGVar->getValueType()),
SGVar->isConstant(), GlobalValue::ExternalLinkage,
/*init*/ nullptr, SGVar->getName(),
/*insertbefore*/ nullptr, SGVar->getThreadLocalMode(),
SGVar->getType()->getAddressSpace());
NewDGV->setAlignment(SGVar->getAlignment());
return NewDGV;
}
static Constant *getGlobalString(LLVMContext &C, Module &M,
const StringRef &str) {
Constant *strArray = ConstantDataArray::getString(C, str);
GlobalVariable *strVar =
new GlobalVariable(M, strArray->getType(), true,
GlobalValue::PrivateLinkage, strArray, "");
strVar->setUnnamedAddr(true);
strVar->setAlignment(1);
std::vector<Value *> params;
params.push_back(ConstantInt::get(TypeBuilder<types::i<32>, true>::get(C), 0));
params.push_back(ConstantInt::get(TypeBuilder<types::i<32>, true>::get(C), 0));
return ConstantExpr::getInBoundsGetElementPtr(strVar, params);
}
void SancusModuleCreator::createFunctionTable(Module& m)
{
LLVMContext& ctx = m.getContext();
// struct SmFunctionInfo
// {
// void* address;
// unsigned arg_length;
// unsigned ret_regs;
// };
Type* funcInfoFields[] = {voidPtrTy, wordTy, wordTy};
StructType* funcInfoTy = StructType::get(ctx, funcInfoFields,
/*isPacked=*/true);
// create a global SM function table for every SM and initialize it
// initializers for the funcs[] array.
// map from section name to initializer
std::map<std::string, std::vector<Constant*>> funcsEls;
for (Function* f : entries)
{
SancusModuleInfo info = getSancusModuleInfo(f);
assert(info.isEntry && "Asking function table for non-entry");
// initializer for the SmFunctionInfo struct
FunctionCcInfo ccInfo(f);
Constant* funcFields[] = {ConstantExpr::getBitCast(f, voidPtrTy),
ConstantInt::get(wordTy, ccInfo.argsLength),
ConstantInt::get(wordTy, ccInfo.retRegsUsage)};
funcsEls[info.getTableSection()]
.push_back(ConstantStruct::get(funcInfoTy, funcFields));
}
for (const auto& it : funcsEls)
{
// struct SmFunctionInfo funcs[];
ArrayType* funcsTy = ArrayType::get(funcInfoTy, it.second.size());
Constant* funcsInit = ConstantArray::get(funcsTy, it.second);
GlobalVariable* table =
new GlobalVariable(m, funcsTy, /*isConstant=*/true,
GlobalVariable::InternalLinkage, funcsInit);
table->setSection(it.first);
table->setAlignment(2);
}
}
void AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I) {
// Array allocations are probably not worth handling, since an allocation of
// the array type is the canonical form.
if (!I.isStaticAlloca() || I.isArrayAllocation())
return;
IRBuilder<> Builder(&I);
// First try to replace the alloca with a vector
Type *AllocaTy = I.getAllocatedType();
DEBUG(dbgs() << "Trying to promote " << I << '\n');
if (tryPromoteAllocaToVector(&I))
return;
DEBUG(dbgs() << " alloca is not a candidate for vectorization.\n");
const Function &ContainingFunction = *I.getParent()->getParent();
// FIXME: We should also try to get this value from the reqd_work_group_size
// function attribute if it is available.
unsigned WorkGroupSize = AMDGPU::getMaximumWorkGroupSize(ContainingFunction);
int AllocaSize =
WorkGroupSize * Mod->getDataLayout().getTypeAllocSize(AllocaTy);
if (AllocaSize > LocalMemAvailable) {
DEBUG(dbgs() << " Not enough local memory to promote alloca.\n");
return;
}
std::vector<Value*> WorkList;
if (!collectUsesWithPtrTypes(&I, WorkList)) {
DEBUG(dbgs() << " Do not know how to convert all uses\n");
return;
}
DEBUG(dbgs() << "Promoting alloca to local memory\n");
LocalMemAvailable -= AllocaSize;
Function *F = I.getParent()->getParent();
Type *GVTy = ArrayType::get(I.getAllocatedType(), WorkGroupSize);
GlobalVariable *GV = new GlobalVariable(
*Mod, GVTy, false, GlobalValue::InternalLinkage,
UndefValue::get(GVTy),
Twine(F->getName()) + Twine('.') + I.getName(),
nullptr,
GlobalVariable::NotThreadLocal,
AMDGPUAS::LOCAL_ADDRESS);
GV->setUnnamedAddr(true);
GV->setAlignment(I.getAlignment());
Value *TCntY, *TCntZ;
std::tie(TCntY, TCntZ) = getLocalSizeYZ(Builder);
Value *TIdX = getWorkitemID(Builder, 0);
Value *TIdY = getWorkitemID(Builder, 1);
Value *TIdZ = getWorkitemID(Builder, 2);
Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ, "", true, true);
Tmp0 = Builder.CreateMul(Tmp0, TIdX);
Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ, "", true, true);
Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
TID = Builder.CreateAdd(TID, TIdZ);
Value *Indices[] = {
Constant::getNullValue(Type::getInt32Ty(Mod->getContext())),
TID
};
Value *Offset = Builder.CreateInBoundsGEP(GVTy, GV, Indices);
I.mutateType(Offset->getType());
I.replaceAllUsesWith(Offset);
I.eraseFromParent();
for (Value *V : WorkList) {
CallInst *Call = dyn_cast<CallInst>(V);
if (!Call) {
Type *EltTy = V->getType()->getPointerElementType();
PointerType *NewTy = PointerType::get(EltTy, AMDGPUAS::LOCAL_ADDRESS);
// The operand's value should be corrected on its own.
if (isa<AddrSpaceCastInst>(V))
continue;
// FIXME: It doesn't really make sense to try to do this for all
// instructions.
V->mutateType(NewTy);
continue;
}
IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(Call);
if (!Intr) {
// FIXME: What is this for? It doesn't make sense to promote arbitrary
// function calls. If the call is to a defined function that can also be
// promoted, we should be able to do this once that function is also
// rewritten.
std::vector<Type*> ArgTypes;
for (unsigned ArgIdx = 0, ArgEnd = Call->getNumArgOperands();
ArgIdx != ArgEnd; ++ArgIdx) {
ArgTypes.push_back(Call->getArgOperand(ArgIdx)->getType());
}
Function *F = Call->getCalledFunction();
FunctionType *NewType = FunctionType::get(Call->getType(), ArgTypes,
F->isVarArg());
Constant *C = Mod->getOrInsertFunction((F->getName() + ".local").str(),
NewType, F->getAttributes());
Function *NewF = cast<Function>(C);
Call->setCalledFunction(NewF);
continue;
}
Builder.SetInsertPoint(Intr);
switch (Intr->getIntrinsicID()) {
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
// These intrinsics are for address space 0 only
Intr->eraseFromParent();
continue;
case Intrinsic::memcpy: {
MemCpyInst *MemCpy = cast<MemCpyInst>(Intr);
Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getRawSource(),
MemCpy->getLength(), MemCpy->getAlignment(),
MemCpy->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memmove: {
MemMoveInst *MemMove = cast<MemMoveInst>(Intr);
Builder.CreateMemMove(MemMove->getRawDest(), MemMove->getRawSource(),
MemMove->getLength(), MemMove->getAlignment(),
MemMove->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memset: {
MemSetInst *MemSet = cast<MemSetInst>(Intr);
Builder.CreateMemSet(MemSet->getRawDest(), MemSet->getValue(),
MemSet->getLength(), MemSet->getAlignment(),
MemSet->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::invariant_start:
case Intrinsic::invariant_end:
case Intrinsic::invariant_group_barrier:
Intr->eraseFromParent();
// FIXME: I think the invariant marker should still theoretically apply,
// but the intrinsics need to be changed to accept pointers with any
// address space.
continue;
case Intrinsic::objectsize: {
Value *Src = Intr->getOperand(0);
Type *SrcTy = Src->getType()->getPointerElementType();
Function *ObjectSize = Intrinsic::getDeclaration(Mod,
Intrinsic::objectsize,
{ Intr->getType(), PointerType::get(SrcTy, AMDGPUAS::LOCAL_ADDRESS) }
);
CallInst *NewCall
= Builder.CreateCall(ObjectSize, { Src, Intr->getOperand(1) });
Intr->replaceAllUsesWith(NewCall);
Intr->eraseFromParent();
continue;
}
default:
Intr->dump();
llvm_unreachable("Don't know how to promote alloca intrinsic use.");
}
}
}
bool LowerEmuTLS::addEmuTlsVar(Module &M, const GlobalVariable *GV) {
LLVMContext &C = M.getContext();
PointerType *VoidPtrType = Type::getInt8PtrTy(C);
std::string EmuTlsVarName = ("__emutls_v." + GV->getName()).str();
GlobalVariable *EmuTlsVar = M.getNamedGlobal(EmuTlsVarName);
if (EmuTlsVar)
return false; // It has been added before.
const DataLayout &DL = M.getDataLayout();
Constant *NullPtr = ConstantPointerNull::get(VoidPtrType);
// Get non-zero initializer from GV's initializer.
const Constant *InitValue = nullptr;
if (GV->hasInitializer()) {
InitValue = GV->getInitializer();
const ConstantInt *InitIntValue = dyn_cast<ConstantInt>(InitValue);
// When GV's init value is all 0, omit the EmuTlsTmplVar and let
// the emutls library function to reset newly allocated TLS variables.
if (isa<ConstantAggregateZero>(InitValue) ||
(InitIntValue && InitIntValue->isZero()))
InitValue = nullptr;
}
// Create the __emutls_v. symbol, whose type has 4 fields:
// word size; // size of GV in bytes
// word align; // alignment of GV
// void *ptr; // initialized to 0; set at run time per thread.
// void *templ; // 0 or point to __emutls_t.*
// sizeof(word) should be the same as sizeof(void*) on target.
IntegerType *WordType = DL.getIntPtrType(C);
PointerType *InitPtrType = InitValue ?
PointerType::getUnqual(InitValue->getType()) : VoidPtrType;
Type *ElementTypes[4] = {WordType, WordType, VoidPtrType, InitPtrType};
ArrayRef<Type*> ElementTypeArray(ElementTypes, 4);
StructType *EmuTlsVarType = StructType::create(ElementTypeArray);
EmuTlsVar = cast<GlobalVariable>(
M.getOrInsertGlobal(EmuTlsVarName, EmuTlsVarType));
copyLinkageVisibility(M, GV, EmuTlsVar);
// Define "__emutls_t.*" and "__emutls_v.*" only if GV is defined.
if (!GV->hasInitializer())
return true;
Type *GVType = GV->getValueType();
unsigned GVAlignment = GV->getAlignment();
if (!GVAlignment) {
// When LLVM IL declares a variable without alignment, use
// the ABI default alignment for the type.
GVAlignment = DL.getABITypeAlignment(GVType);
}
// Define "__emutls_t.*" if there is InitValue
GlobalVariable *EmuTlsTmplVar = nullptr;
if (InitValue) {
std::string EmuTlsTmplName = ("__emutls_t." + GV->getName()).str();
EmuTlsTmplVar = dyn_cast_or_null<GlobalVariable>(
M.getOrInsertGlobal(EmuTlsTmplName, GVType));
assert(EmuTlsTmplVar && "Failed to create emualted TLS initializer");
EmuTlsTmplVar->setConstant(true);
EmuTlsTmplVar->setInitializer(const_cast<Constant*>(InitValue));
EmuTlsTmplVar->setAlignment(GVAlignment);
copyLinkageVisibility(M, GV, EmuTlsTmplVar);
}
// Define "__emutls_v.*" with initializer and alignment.
Constant *ElementValues[4] = {
ConstantInt::get(WordType, DL.getTypeStoreSize(GVType)),
ConstantInt::get(WordType, GVAlignment),
NullPtr, EmuTlsTmplVar ? EmuTlsTmplVar : NullPtr
};
ArrayRef<Constant*> ElementValueArray(ElementValues, 4);
EmuTlsVar->setInitializer(
ConstantStruct::get(EmuTlsVarType, ElementValueArray));
unsigned MaxAlignment = std::max(
DL.getABITypeAlignment(WordType),
DL.getABITypeAlignment(VoidPtrType));
EmuTlsVar->setAlignment(MaxAlignment);
return true;
}
bool SanitizerCoverageModule::runOnModule(Module &M) {
if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
return false;
C = &(M.getContext());
DL = &M.getDataLayout();
CurModule = &M;
HasSancovGuardsSection = false;
IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
IntptrPtrTy = PointerType::getUnqual(IntptrTy);
Type *VoidTy = Type::getVoidTy(*C);
IRBuilder<> IRB(*C);
Type *Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty());
Int64PtrTy = PointerType::getUnqual(IRB.getInt64Ty());
Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
Int64Ty = IRB.getInt64Ty();
Int32Ty = IRB.getInt32Ty();
SanCovFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovName, VoidTy, Int32PtrTy, nullptr));
SanCovWithCheckFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovWithCheckName, VoidTy, Int32PtrTy, nullptr));
SanCovTracePCIndir = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy, nullptr));
SanCovIndirCallFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovIndirCallName, VoidTy, IntptrTy, IntptrTy, nullptr));
SanCovTraceCmpFunction[0] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceCmp1, VoidTy, IRB.getInt8Ty(), IRB.getInt8Ty(), nullptr));
SanCovTraceCmpFunction[1] = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTraceCmp2, VoidTy, IRB.getInt16Ty(),
IRB.getInt16Ty(), nullptr));
SanCovTraceCmpFunction[2] = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTraceCmp4, VoidTy, IRB.getInt32Ty(),
IRB.getInt32Ty(), nullptr));
SanCovTraceCmpFunction[3] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty, nullptr));
SanCovTraceDivFunction[0] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceDiv4, VoidTy, IRB.getInt32Ty(), nullptr));
SanCovTraceDivFunction[1] =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceDiv8, VoidTy, Int64Ty, nullptr));
SanCovTraceGepFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceGep, VoidTy, IntptrTy, nullptr));
SanCovTraceSwitchFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy, nullptr));
// We insert an empty inline asm after cov callbacks to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
/*hasSideEffects=*/true);
SanCovTracePC = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTracePCName, VoidTy, nullptr));
SanCovTracePCGuard = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
SanCovTracePCGuardName, VoidTy, Int32PtrTy, nullptr));
SanCovTraceEnter = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTraceEnterName, VoidTy, Int32PtrTy, nullptr));
SanCovTraceBB = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(SanCovTraceBBName, VoidTy, Int32PtrTy, nullptr));
// At this point we create a dummy array of guards because we don't
// know how many elements we will need.
Type *Int32Ty = IRB.getInt32Ty();
Type *Int8Ty = IRB.getInt8Ty();
if (!Options.TracePCGuard)
GuardArray =
new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
if (Options.Use8bitCounters)
EightBitCounterArray =
new GlobalVariable(M, Int8Ty, false, GlobalVariable::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
for (auto &F : M)
runOnFunction(F);
auto N = NumberOfInstrumentedBlocks();
GlobalVariable *RealGuardArray = nullptr;
if (!Options.TracePCGuard) {
// Now we know how many elements we need. Create an array of guards
// with one extra element at the beginning for the size.
Type *Int32ArrayNTy = ArrayType::get(Int32Ty, N + 1);
RealGuardArray = new GlobalVariable(
M, Int32ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int32ArrayNTy), "__sancov_gen_cov");
// Replace the dummy array with the real one.
GuardArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealGuardArray, Int32PtrTy));
GuardArray->eraseFromParent();
}
GlobalVariable *RealEightBitCounterArray;
if (Options.Use8bitCounters) {
// Make sure the array is 16-aligned.
static const int CounterAlignment = 16;
Type *Int8ArrayNTy = ArrayType::get(Int8Ty, alignTo(N, CounterAlignment));
RealEightBitCounterArray = new GlobalVariable(
M, Int8ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int8ArrayNTy), "__sancov_gen_cov_counter");
RealEightBitCounterArray->setAlignment(CounterAlignment);
EightBitCounterArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy));
EightBitCounterArray->eraseFromParent();
}
// Create variable for module (compilation unit) name
Constant *ModNameStrConst =
ConstantDataArray::getString(M.getContext(), M.getName(), true);
GlobalVariable *ModuleName =
new GlobalVariable(M, ModNameStrConst->getType(), true,
GlobalValue::PrivateLinkage, ModNameStrConst);
if (Options.TracePCGuard) {
if (HasSancovGuardsSection) {
Function *CtorFunc;
std::string SectionName(SanCovTracePCGuardSection);
GlobalVariable *Bounds[2];
const char *Prefix[2] = {"__start_", "__stop_"};
for (int i = 0; i < 2; i++) {
Bounds[i] = new GlobalVariable(M, Int32PtrTy, false,
GlobalVariable::ExternalLinkage, nullptr,
Prefix[i] + SectionName);
Bounds[i]->setVisibility(GlobalValue::HiddenVisibility);
}
std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
M, SanCovModuleCtorName, SanCovTracePCGuardInitName,
{Int32PtrTy, Int32PtrTy},
{IRB.CreatePointerCast(Bounds[0], Int32PtrTy),
IRB.CreatePointerCast(Bounds[1], Int32PtrTy)});
appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);
}
} else if (!Options.TracePC) {
Function *CtorFunc;
std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
M, SanCovModuleCtorName, SanCovModuleInitName,
{Int32PtrTy, IntptrTy, Int8PtrTy, Int8PtrTy},
{IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
ConstantInt::get(IntptrTy, N),
Options.Use8bitCounters
? IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy)
: Constant::getNullValue(Int8PtrTy),
IRB.CreatePointerCast(ModuleName, Int8PtrTy)});
appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);
}
return true;
}
/// emit_global_to_llvm - Emit the specified VAR_DECL or aggregate CONST_DECL to
/// LLVM as a global variable. This function implements the end of
/// assemble_variable.
void emit_global_to_llvm(tree decl) {
if (errorcount || sorrycount) return;
// FIXME: Support alignment on globals: DECL_ALIGN.
// FIXME: DECL_PRESERVE_P indicates the var is marked with attribute 'used'.
// Global register variables don't turn into LLVM GlobalVariables.
if (TREE_CODE(decl) == VAR_DECL && DECL_REGISTER(decl))
return;
timevar_push(TV_LLVM_GLOBALS);
// Get or create the global variable now.
GlobalVariable *GV = cast<GlobalVariable>(DECL_LLVM(decl));
// Convert the initializer over.
Constant *Init;
if (DECL_INITIAL(decl) == 0 || DECL_INITIAL(decl) == error_mark_node) {
// This global should be zero initialized. Reconvert the type in case the
// forward def of the global and the real def differ in type (e.g. declared
// as 'int A[]', and defined as 'int A[100]').
Init = Constant::getNullValue(ConvertType(TREE_TYPE(decl)));
} else {
assert((TREE_CONSTANT(DECL_INITIAL(decl)) ||
TREE_CODE(DECL_INITIAL(decl)) == STRING_CST) &&
"Global initializer should be constant!");
// Temporarily set an initializer for the global, so we don't infinitely
// recurse. If we don't do this, we can hit cases where we see "oh a global
// with an initializer hasn't been initialized yet, call emit_global_to_llvm
// on it". When constructing the initializer it might refer to itself.
// this can happen for things like void *G = &G;
//
GV->setInitializer(UndefValue::get(GV->getType()->getElementType()));
Init = TreeConstantToLLVM::Convert(DECL_INITIAL(decl));
}
// If we had a forward definition that has a type that disagrees with our
// initializer, insert a cast now. This sort of thing occurs when we have a
// global union, and the LLVM type followed a union initializer that is
// different from the union element used for the type.
if (GV->getType()->getElementType() != Init->getType()) {
GV->removeFromParent();
GlobalVariable *NGV = new GlobalVariable(Init->getType(), GV->isConstant(),
GlobalValue::ExternalLinkage, 0,
GV->getName(), TheModule);
GV->replaceAllUsesWith(ConstantExpr::getBitCast(NGV, GV->getType()));
delete GV;
SET_DECL_LLVM(decl, NGV);
GV = NGV;
}
// Set the initializer.
GV->setInitializer(Init);
// Set thread local (TLS)
if (TREE_CODE(decl) == VAR_DECL && DECL_THREAD_LOCAL(decl))
GV->setThreadLocal(true);
// Set the linkage.
if (!TREE_PUBLIC(decl)) {
GV->setLinkage(GlobalValue::InternalLinkage);
} else if (DECL_WEAK(decl) || DECL_ONE_ONLY(decl) ||
(DECL_COMMON(decl) && // DECL_COMMON is only meaningful if no init
(!DECL_INITIAL(decl) || DECL_INITIAL(decl) == error_mark_node))) {
// llvm-gcc also includes DECL_VIRTUAL_P here.
GV->setLinkage(GlobalValue::WeakLinkage);
} else if (DECL_COMDAT(decl)) {
GV->setLinkage(GlobalValue::LinkOnceLinkage);
}
#ifdef TARGET_ADJUST_LLVM_LINKAGE
TARGET_ADJUST_LLVM_LINKAGE(GV,decl);
#endif /* TARGET_ADJUST_LLVM_LINKAGE */
// Handle visibility style
if (TREE_PUBLIC(decl)) {
if (DECL_VISIBILITY(decl) == VISIBILITY_HIDDEN)
GV->setVisibility(GlobalValue::HiddenVisibility);
else if (DECL_VISIBILITY(decl) == VISIBILITY_PROTECTED)
GV->setVisibility(GlobalValue::ProtectedVisibility);
}
// Set the section for the global.
if (TREE_CODE(decl) == VAR_DECL || TREE_CODE(decl) == CONST_DECL) {
if (DECL_SECTION_NAME(decl)) {
GV->setSection(TREE_STRING_POINTER(DECL_SECTION_NAME(decl)));
#ifdef LLVM_IMPLICIT_TARGET_GLOBAL_VAR_SECTION
} else if (const char *Section =
LLVM_IMPLICIT_TARGET_GLOBAL_VAR_SECTION(decl)) {
GV->setSection(Section);
#endif
}
// Set the alignment for the global if one of the following condition is met
// 1) DECL_ALIGN_UNIT does not match alignment as per ABI specification
// 2) DECL_ALIGN is set by user.
if (DECL_ALIGN_UNIT(decl)) {
unsigned TargetAlign = getTargetData().getABITypeAlignment(GV->getType()->getElementType());
if (DECL_USER_ALIGN(decl) || TargetAlign != DECL_ALIGN_UNIT(decl))
GV->setAlignment(DECL_ALIGN_UNIT(decl));
}
// Handle used decls
if (DECL_PRESERVE_P (decl)) {
const Type *SBP= PointerType::get(Type::Int8Ty);
AttributeUsedGlobals.push_back(ConstantExpr::getBitCast(GV, SBP));
}
// Add annotate attributes for globals
if (DECL_ATTRIBUTES(decl))
AddAnnotateAttrsToGlobal(GV, decl);
}
if (TheDebugInfo) TheDebugInfo->EmitGlobalVariable(GV, decl);
timevar_pop(TV_LLVM_GLOBALS);
}
// FIXME: Should try to pick the most likely to be profitable allocas first.
bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
// Array allocations are probably not worth handling, since an allocation of
// the array type is the canonical form.
if (!I.isStaticAlloca() || I.isArrayAllocation())
return false;
IRBuilder<> Builder(&I);
// First try to replace the alloca with a vector
Type *AllocaTy = I.getAllocatedType();
DEBUG(dbgs() << "Trying to promote " << I << '\n');
if (tryPromoteAllocaToVector(&I, AS))
return true; // Promoted to vector.
const Function &ContainingFunction = *I.getParent()->getParent();
CallingConv::ID CC = ContainingFunction.getCallingConv();
// Don't promote the alloca to LDS for shader calling conventions as the work
// item ID intrinsics are not supported for these calling conventions.
// Furthermore not all LDS is available for some of the stages.
switch (CC) {
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
break;
default:
DEBUG(dbgs() << " promote alloca to LDS not supported with calling convention.\n");
return false;
}
// Not likely to have sufficient local memory for promotion.
if (!SufficientLDS)
return false;
const AMDGPUSubtarget &ST =
TM->getSubtarget<AMDGPUSubtarget>(ContainingFunction);
unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(ContainingFunction).second;
const DataLayout &DL = Mod->getDataLayout();
unsigned Align = I.getAlignment();
if (Align == 0)
Align = DL.getABITypeAlignment(I.getAllocatedType());
// FIXME: This computed padding is likely wrong since it depends on inverse
// usage order.
//
// FIXME: It is also possible that if we're allowed to use all of the memory
// could could end up using more than the maximum due to alignment padding.
uint32_t NewSize = alignTo(CurrentLocalMemUsage, Align);
uint32_t AllocSize = WorkGroupSize * DL.getTypeAllocSize(AllocaTy);
NewSize += AllocSize;
if (NewSize > LocalMemLimit) {
DEBUG(dbgs() << " " << AllocSize
<< " bytes of local memory not available to promote\n");
return false;
}
CurrentLocalMemUsage = NewSize;
std::vector<Value*> WorkList;
if (!collectUsesWithPtrTypes(&I, &I, WorkList)) {
DEBUG(dbgs() << " Do not know how to convert all uses\n");
return false;
}
DEBUG(dbgs() << "Promoting alloca to local memory\n");
Function *F = I.getParent()->getParent();
Type *GVTy = ArrayType::get(I.getAllocatedType(), WorkGroupSize);
GlobalVariable *GV = new GlobalVariable(
*Mod, GVTy, false, GlobalValue::InternalLinkage,
UndefValue::get(GVTy),
Twine(F->getName()) + Twine('.') + I.getName(),
nullptr,
GlobalVariable::NotThreadLocal,
AS.LOCAL_ADDRESS);
GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
GV->setAlignment(I.getAlignment());
Value *TCntY, *TCntZ;
std::tie(TCntY, TCntZ) = getLocalSizeYZ(Builder);
Value *TIdX = getWorkitemID(Builder, 0);
Value *TIdY = getWorkitemID(Builder, 1);
Value *TIdZ = getWorkitemID(Builder, 2);
Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ, "", true, true);
Tmp0 = Builder.CreateMul(Tmp0, TIdX);
Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ, "", true, true);
Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
TID = Builder.CreateAdd(TID, TIdZ);
Value *Indices[] = {
Constant::getNullValue(Type::getInt32Ty(Mod->getContext())),
TID
};
Value *Offset = Builder.CreateInBoundsGEP(GVTy, GV, Indices);
I.mutateType(Offset->getType());
I.replaceAllUsesWith(Offset);
I.eraseFromParent();
for (Value *V : WorkList) {
CallInst *Call = dyn_cast<CallInst>(V);
if (!Call) {
if (ICmpInst *CI = dyn_cast<ICmpInst>(V)) {
Value *Src0 = CI->getOperand(0);
Type *EltTy = Src0->getType()->getPointerElementType();
PointerType *NewTy = PointerType::get(EltTy, AS.LOCAL_ADDRESS);
if (isa<ConstantPointerNull>(CI->getOperand(0)))
CI->setOperand(0, ConstantPointerNull::get(NewTy));
if (isa<ConstantPointerNull>(CI->getOperand(1)))
CI->setOperand(1, ConstantPointerNull::get(NewTy));
continue;
}
// The operand's value should be corrected on its own and we don't want to
// touch the users.
if (isa<AddrSpaceCastInst>(V))
continue;
Type *EltTy = V->getType()->getPointerElementType();
PointerType *NewTy = PointerType::get(EltTy, AS.LOCAL_ADDRESS);
// FIXME: It doesn't really make sense to try to do this for all
// instructions.
V->mutateType(NewTy);
// Adjust the types of any constant operands.
if (SelectInst *SI = dyn_cast<SelectInst>(V)) {
if (isa<ConstantPointerNull>(SI->getOperand(1)))
SI->setOperand(1, ConstantPointerNull::get(NewTy));
if (isa<ConstantPointerNull>(SI->getOperand(2)))
SI->setOperand(2, ConstantPointerNull::get(NewTy));
} else if (PHINode *Phi = dyn_cast<PHINode>(V)) {
for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
if (isa<ConstantPointerNull>(Phi->getIncomingValue(I)))
Phi->setIncomingValue(I, ConstantPointerNull::get(NewTy));
}
}
continue;
}
IntrinsicInst *Intr = cast<IntrinsicInst>(Call);
Builder.SetInsertPoint(Intr);
switch (Intr->getIntrinsicID()) {
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
// These intrinsics are for address space 0 only
Intr->eraseFromParent();
continue;
case Intrinsic::memcpy: {
MemCpyInst *MemCpy = cast<MemCpyInst>(Intr);
Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getDestAlignment(),
MemCpy->getRawSource(), MemCpy->getSourceAlignment(),
MemCpy->getLength(), MemCpy->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memmove: {
MemMoveInst *MemMove = cast<MemMoveInst>(Intr);
Builder.CreateMemMove(MemMove->getRawDest(), MemMove->getDestAlignment(),
MemMove->getRawSource(), MemMove->getSourceAlignment(),
MemMove->getLength(), MemMove->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::memset: {
MemSetInst *MemSet = cast<MemSetInst>(Intr);
Builder.CreateMemSet(MemSet->getRawDest(), MemSet->getValue(),
MemSet->getLength(), MemSet->getDestAlignment(),
MemSet->isVolatile());
Intr->eraseFromParent();
continue;
}
case Intrinsic::invariant_start:
case Intrinsic::invariant_end:
case Intrinsic::invariant_group_barrier:
Intr->eraseFromParent();
// FIXME: I think the invariant marker should still theoretically apply,
// but the intrinsics need to be changed to accept pointers with any
// address space.
continue;
case Intrinsic::objectsize: {
Value *Src = Intr->getOperand(0);
Type *SrcTy = Src->getType()->getPointerElementType();
Function *ObjectSize = Intrinsic::getDeclaration(Mod,
Intrinsic::objectsize,
{ Intr->getType(), PointerType::get(SrcTy, AS.LOCAL_ADDRESS) }
);
CallInst *NewCall = Builder.CreateCall(
ObjectSize, {Src, Intr->getOperand(1), Intr->getOperand(2)});
Intr->replaceAllUsesWith(NewCall);
Intr->eraseFromParent();
continue;
}
default:
Intr->print(errs());
llvm_unreachable("Don't know how to promote alloca intrinsic use.");
}
}
return true;
}
bool SanitizerCoverageModule::runOnModule(Module &M) {
if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
return false;
C = &(M.getContext());
DL = &M.getDataLayout();
IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
Type *VoidTy = Type::getVoidTy(*C);
IRBuilder<> IRB(*C);
Type *Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty());
Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
Int64Ty = IRB.getInt64Ty();
SanCovFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovName, VoidTy, Int32PtrTy, nullptr));
SanCovWithCheckFunction = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovWithCheckName, VoidTy, Int32PtrTy, nullptr));
SanCovIndirCallFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovIndirCallName, VoidTy, IntptrTy, IntptrTy, nullptr));
SanCovTraceCmpFunction =
checkSanitizerInterfaceFunction(M.getOrInsertFunction(
kSanCovTraceCmp, VoidTy, Int64Ty, Int64Ty, Int64Ty, nullptr));
// We insert an empty inline asm after cov callbacks to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
/*hasSideEffects=*/true);
if (Options.TraceBB) {
SanCovTraceEnter = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovTraceEnter, VoidTy, Int32PtrTy, nullptr));
SanCovTraceBB = checkSanitizerInterfaceFunction(
M.getOrInsertFunction(kSanCovTraceBB, VoidTy, Int32PtrTy, nullptr));
}
// At this point we create a dummy array of guards because we don't
// know how many elements we will need.
Type *Int32Ty = IRB.getInt32Ty();
Type *Int8Ty = IRB.getInt8Ty();
GuardArray =
new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
if (Options.Use8bitCounters)
EightBitCounterArray =
new GlobalVariable(M, Int8Ty, false, GlobalVariable::ExternalLinkage,
nullptr, "__sancov_gen_cov_tmp");
for (auto &F : M)
runOnFunction(F);
auto N = NumberOfInstrumentedBlocks();
// Now we know how many elements we need. Create an array of guards
// with one extra element at the beginning for the size.
Type *Int32ArrayNTy = ArrayType::get(Int32Ty, N + 1);
GlobalVariable *RealGuardArray = new GlobalVariable(
M, Int32ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int32ArrayNTy), "__sancov_gen_cov");
// Replace the dummy array with the real one.
GuardArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealGuardArray, Int32PtrTy));
GuardArray->eraseFromParent();
GlobalVariable *RealEightBitCounterArray;
if (Options.Use8bitCounters) {
// Make sure the array is 16-aligned.
static const int kCounterAlignment = 16;
Type *Int8ArrayNTy =
ArrayType::get(Int8Ty, RoundUpToAlignment(N, kCounterAlignment));
RealEightBitCounterArray = new GlobalVariable(
M, Int8ArrayNTy, false, GlobalValue::PrivateLinkage,
Constant::getNullValue(Int8ArrayNTy), "__sancov_gen_cov_counter");
RealEightBitCounterArray->setAlignment(kCounterAlignment);
EightBitCounterArray->replaceAllUsesWith(
IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy));
EightBitCounterArray->eraseFromParent();
}
// Create variable for module (compilation unit) name
Constant *ModNameStrConst =
ConstantDataArray::getString(M.getContext(), M.getName(), true);
GlobalVariable *ModuleName =
new GlobalVariable(M, ModNameStrConst->getType(), true,
GlobalValue::PrivateLinkage, ModNameStrConst);
Function *CtorFunc;
std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
M, kSanCovModuleCtorName, kSanCovModuleInitName,
{Int32PtrTy, IntptrTy, Int8PtrTy, Int8PtrTy},
{IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
ConstantInt::get(IntptrTy, N),
Options.Use8bitCounters
? IRB.CreatePointerCast(RealEightBitCounterArray, Int8PtrTy)
: Constant::getNullValue(Int8PtrTy),
IRB.CreatePointerCast(ModuleName, Int8PtrTy)});
appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
return true;
}
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
GlobalVariable *Name = Inc->getName();
auto It = RegionCounters.find(Name);
if (It != RegionCounters.end())
return It->second;
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
// only one copy of counters of the COMDAT function will be emitted after
// linking.
Function *Fn = Inc->getParent()->getParent();
Comdat *ProfileVarsComdat = nullptr;
if (Fn->hasComdat())
ProfileVarsComdat = M->getOrInsertComdat(StringRef(getVarName(Inc, "vars")));
Name->setSection(getNameSection());
Name->setAlignment(1);
Name->setComdat(ProfileVarsComdat);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
Constant::getNullValue(CounterTy),
getVarName(Inc, "counters"));
Counters->setVisibility(Name->getVisibility());
Counters->setSection(getCountersSection());
Counters->setAlignment(8);
Counters->setComdat(ProfileVarsComdat);
RegionCounters[Inc->getName()] = Counters;
// Create data variable.
auto *NameArrayTy = Name->getType()->getPointerElementType();
auto *Int32Ty = Type::getInt32Ty(Ctx);
auto *Int64Ty = Type::getInt64Ty(Ctx);
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Constant *DataVals[] = {
ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
ConstantInt::get(Int32Ty, NumCounters),
ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
ConstantExpr::getBitCast(Name, Int8PtrTy),
ConstantExpr::getBitCast(Counters, Int64PtrTy)};
auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
getVarName(Inc, "data"));
Data->setVisibility(Name->getVisibility());
Data->setSection(getDataSection());
Data->setAlignment(8);
Data->setComdat(ProfileVarsComdat);
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
return Counters;
}
Module *llvm::CloneModule(const Module *M,
ValueToValueMapTy &VMap) {
// First off, we need to create the new module...
Module *New = new Module(M->getModuleIdentifier(), M->getContext());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Copy all of the type symbol table entries over.
const TypeSymbolTable &TST = M->getTypeSymbolTable();
for (TypeSymbolTable::const_iterator TI = TST.begin(), TE = TST.end();
TI != TE; ++TI)
New->addTypeName(TI->first, TI->second);
// Copy all of the dependent libraries over.
for (Module::lib_iterator I = M->lib_begin(), E = M->lib_end(); I != E; ++I)
New->addLibrary(*I);
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the VMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(*New,
I->getType()->getElementType(),
false,
GlobalValue::ExternalLinkage, 0,
I->getName());
GV->setAlignment(I->getAlignment());
VMap[I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getType()->getElementType()),
GlobalValue::ExternalLinkage, I->getName(), New);
NF->copyAttributesFrom(I);
VMap[I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I)
VMap[I] = new GlobalAlias(I->getType(), GlobalAlias::ExternalLinkage,
I->getName(), NULL, New);
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(VMap[I]);
if (I->hasInitializer())
GV->setInitializer(cast<Constant>(MapValue(I->getInitializer(),
VMap)));
GV->setLinkage(I->getLinkage());
GV->setThreadLocal(I->isThreadLocal());
GV->setConstant(I->isConstant());
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(VMap[I]);
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
VMap[J] = DestI++;
}
SmallVector<ReturnInst*, 8> Returns; // Ignore returns cloned.
CloneFunctionInto(F, I, VMap, Returns);
}
F->setLinkage(I->getLinkage());
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
GlobalAlias *GA = cast<GlobalAlias>(VMap[I]);
GA->setLinkage(I->getLinkage());
if (const Constant* C = I->getAliasee())
GA->setAliasee(cast<Constant>(MapValue(C, VMap)));
}
// And named metadata....
for (Module::const_named_metadata_iterator I = M->named_metadata_begin(),
E = M->named_metadata_end(); I != E; ++I) {
const NamedMDNode &NMD = *I;
SmallVector<MDNode*, 4> MDs;
for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i)
MDs.push_back(cast<MDNode>(MapValue(NMD.getOperand(i), VMap)));
NamedMDNode::Create(New->getContext(), NMD.getName(),
MDs.data(), MDs.size(), New);
}
// Update metadata attach with instructions.
for (Module::iterator MI = New->begin(), ME = New->end(); MI != ME; ++MI)
for (Function::iterator FI = MI->begin(), FE = MI->end();
FI != FE; ++FI)
for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
BI != BE; ++BI) {
SmallVector<std::pair<unsigned, MDNode *>, 4 > MDs;
BI->getAllMetadata(MDs);
for (SmallVector<std::pair<unsigned, MDNode *>, 4>::iterator
MDI = MDs.begin(), MDE = MDs.end(); MDI != MDE; ++MDI) {
Value *MappedValue = MapValue(MDI->second, VMap);
if (MDI->second != MappedValue && MappedValue)
BI->setMetadata(MDI->first, cast<MDNode>(MappedValue));
}
}
return New;
}
// This function replaces all global variables with new variables that have
// trailing redzones. It also creates a function that poisons
// redzones and inserts this function into llvm.global_ctors.
bool AddressSanitizer::insertGlobalRedzones(Module &M) {
SmallVector<GlobalVariable *, 16> GlobalsToChange;
for (Module::GlobalListType::iterator G = M.global_begin(),
E = M.global_end(); G != E; ++G) {
if (ShouldInstrumentGlobal(G))
GlobalsToChange.push_back(G);
}
size_t n = GlobalsToChange.size();
if (n == 0) return false;
// A global is described by a structure
// size_t beg;
// size_t size;
// size_t size_with_redzone;
// const char *name;
// size_t has_dynamic_init;
// We initialize an array of such structures and pass it to a run-time call.
StructType *GlobalStructTy = StructType::get(IntptrTy, IntptrTy,
IntptrTy, IntptrTy,
IntptrTy, NULL);
SmallVector<Constant *, 16> Initializers(n), DynamicInit;
IRBuilder<> IRB(CtorInsertBefore);
if (ClInitializers)
FindDynamicInitializers(M);
// The addresses of the first and last dynamically initialized globals in
// this TU. Used in initialization order checking.
Value *FirstDynamic = 0, *LastDynamic = 0;
for (size_t i = 0; i < n; i++) {
GlobalVariable *G = GlobalsToChange[i];
PointerType *PtrTy = cast<PointerType>(G->getType());
Type *Ty = PtrTy->getElementType();
uint64_t SizeInBytes = TD->getTypeAllocSize(Ty);
uint64_t RightRedzoneSize = RedzoneSize +
(RedzoneSize - (SizeInBytes % RedzoneSize));
Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize);
// Determine whether this global should be poisoned in initialization.
bool GlobalHasDynamicInitializer = HasDynamicInitializer(G);
// Don't check initialization order if this global is blacklisted.
GlobalHasDynamicInitializer &= !BL->isInInit(*G);
StructType *NewTy = StructType::get(Ty, RightRedZoneTy, NULL);
Constant *NewInitializer = ConstantStruct::get(
NewTy, G->getInitializer(),
Constant::getNullValue(RightRedZoneTy), NULL);
SmallString<2048> DescriptionOfGlobal = G->getName();
DescriptionOfGlobal += " (";
DescriptionOfGlobal += M.getModuleIdentifier();
DescriptionOfGlobal += ")";
GlobalVariable *Name = createPrivateGlobalForString(M, DescriptionOfGlobal);
// Create a new global variable with enough space for a redzone.
GlobalVariable *NewGlobal = new GlobalVariable(
M, NewTy, G->isConstant(), G->getLinkage(),
NewInitializer, "", G, G->getThreadLocalMode());
NewGlobal->copyAttributesFrom(G);
NewGlobal->setAlignment(RedzoneSize);
Value *Indices2[2];
Indices2[0] = IRB.getInt32(0);
Indices2[1] = IRB.getInt32(0);
G->replaceAllUsesWith(
ConstantExpr::getGetElementPtr(NewGlobal, Indices2, true));
NewGlobal->takeName(G);
G->eraseFromParent();
Initializers[i] = ConstantStruct::get(
GlobalStructTy,
ConstantExpr::getPointerCast(NewGlobal, IntptrTy),
ConstantInt::get(IntptrTy, SizeInBytes),
ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
ConstantExpr::getPointerCast(Name, IntptrTy),
ConstantInt::get(IntptrTy, GlobalHasDynamicInitializer),
NULL);
// Populate the first and last globals declared in this TU.
if (ClInitializers && GlobalHasDynamicInitializer) {
LastDynamic = ConstantExpr::getPointerCast(NewGlobal, IntptrTy);
if (FirstDynamic == 0)
FirstDynamic = LastDynamic;
}
DEBUG(dbgs() << "NEW GLOBAL:\n" << *NewGlobal);
}
ArrayType *ArrayOfGlobalStructTy = ArrayType::get(GlobalStructTy, n);
GlobalVariable *AllGlobals = new GlobalVariable(
M, ArrayOfGlobalStructTy, false, GlobalVariable::PrivateLinkage,
ConstantArray::get(ArrayOfGlobalStructTy, Initializers), "");
// Create calls for poisoning before initializers run and unpoisoning after.
if (ClInitializers && FirstDynamic && LastDynamic)
createInitializerPoisonCalls(M, FirstDynamic, LastDynamic);
Function *AsanRegisterGlobals = checkInterfaceFunction(M.getOrInsertFunction(
kAsanRegisterGlobalsName, IRB.getVoidTy(),
IntptrTy, IntptrTy, NULL));
AsanRegisterGlobals->setLinkage(Function::ExternalLinkage);
IRB.CreateCall2(AsanRegisterGlobals,
IRB.CreatePointerCast(AllGlobals, IntptrTy),
ConstantInt::get(IntptrTy, n));
// We also need to unregister globals at the end, e.g. when a shared library
// gets closed.
Function *AsanDtorFunction = Function::Create(
FunctionType::get(Type::getVoidTy(*C), false),
GlobalValue::InternalLinkage, kAsanModuleDtorName, &M);
BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction);
IRBuilder<> IRB_Dtor(ReturnInst::Create(*C, AsanDtorBB));
Function *AsanUnregisterGlobals =
checkInterfaceFunction(M.getOrInsertFunction(
kAsanUnregisterGlobalsName,
IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage);
IRB_Dtor.CreateCall2(AsanUnregisterGlobals,
IRB.CreatePointerCast(AllGlobals, IntptrTy),
ConstantInt::get(IntptrTy, n));
appendToGlobalDtors(M, AsanDtorFunction, kAsanCtorAndCtorPriority);
DEBUG(dbgs() << M);
return true;
}
// This function replaces all global variables with new variables that have
// trailing redzones. It also creates a function that poisons
// redzones and inserts this function into llvm.global_ctors.
bool AddressSanitizer::insertGlobalRedzones(Module &M) {
SmallVector<GlobalVariable *, 16> GlobalsToChange;
for (Module::GlobalListType::iterator G = M.getGlobalList().begin(),
E = M.getGlobalList().end(); G != E; ++G) {
Type *Ty = cast<PointerType>(G->getType())->getElementType();
DEBUG(dbgs() << "GLOBAL: " << *G);
if (!Ty->isSized()) continue;
if (!G->hasInitializer()) continue;
// Touch only those globals that will not be defined in other modules.
// Don't handle ODR type linkages since other modules may be built w/o asan.
if (G->getLinkage() != GlobalVariable::ExternalLinkage &&
G->getLinkage() != GlobalVariable::PrivateLinkage &&
G->getLinkage() != GlobalVariable::InternalLinkage)
continue;
// Two problems with thread-locals:
// - The address of the main thread's copy can't be computed at link-time.
// - Need to poison all copies, not just the main thread's one.
if (G->isThreadLocal())
continue;
// For now, just ignore this Alloca if the alignment is large.
if (G->getAlignment() > RedzoneSize) continue;
// Ignore all the globals with the names starting with "\01L_OBJC_".
// Many of those are put into the .cstring section. The linker compresses
// that section by removing the spare \0s after the string terminator, so
// our redzones get broken.
if ((G->getName().find("\01L_OBJC_") == 0) ||
(G->getName().find("\01l_OBJC_") == 0)) {
DEBUG(dbgs() << "Ignoring \\01L_OBJC_* global: " << *G);
continue;
}
if (G->hasSection()) {
StringRef Section(G->getSection());
// Ignore the globals from the __OBJC section. The ObjC runtime assumes
// those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
// them.
if ((Section.find("__OBJC,") == 0) ||
(Section.find("__DATA, __objc_") == 0)) {
DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G);
continue;
}
// See http://code.google.com/p/address-sanitizer/issues/detail?id=32
// Constant CFString instances are compiled in the following way:
// -- the string buffer is emitted into
// __TEXT,__cstring,cstring_literals
// -- the constant NSConstantString structure referencing that buffer
// is placed into __DATA,__cfstring
// Therefore there's no point in placing redzones into __DATA,__cfstring.
// Moreover, it causes the linker to crash on OS X 10.7
if (Section.find("__DATA,__cfstring") == 0) {
DEBUG(dbgs() << "Ignoring CFString: " << *G);
continue;
}
}
GlobalsToChange.push_back(G);
}
size_t n = GlobalsToChange.size();
if (n == 0) return false;
// A global is described by a structure
// size_t beg;
// size_t size;
// size_t size_with_redzone;
// const char *name;
// We initialize an array of such structures and pass it to a run-time call.
StructType *GlobalStructTy = StructType::get(IntptrTy, IntptrTy,
IntptrTy, IntptrTy, NULL);
SmallVector<Constant *, 16> Initializers(n);
IRBuilder<> IRB(CtorInsertBefore);
for (size_t i = 0; i < n; i++) {
GlobalVariable *G = GlobalsToChange[i];
PointerType *PtrTy = cast<PointerType>(G->getType());
Type *Ty = PtrTy->getElementType();
uint64_t SizeInBytes = TD->getTypeStoreSizeInBits(Ty) / 8;
uint64_t RightRedzoneSize = RedzoneSize +
(RedzoneSize - (SizeInBytes % RedzoneSize));
Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize);
StructType *NewTy = StructType::get(Ty, RightRedZoneTy, NULL);
Constant *NewInitializer = ConstantStruct::get(
NewTy, G->getInitializer(),
Constant::getNullValue(RightRedZoneTy), NULL);
SmallString<2048> DescriptionOfGlobal = G->getName();
DescriptionOfGlobal += " (";
DescriptionOfGlobal += M.getModuleIdentifier();
DescriptionOfGlobal += ")";
GlobalVariable *Name = createPrivateGlobalForString(M, DescriptionOfGlobal);
// Create a new global variable with enough space for a redzone.
GlobalVariable *NewGlobal = new GlobalVariable(
M, NewTy, G->isConstant(), G->getLinkage(),
NewInitializer, "", G, G->isThreadLocal());
NewGlobal->copyAttributesFrom(G);
NewGlobal->setAlignment(RedzoneSize);
Value *Indices2[2];
Indices2[0] = IRB.getInt32(0);
Indices2[1] = IRB.getInt32(0);
G->replaceAllUsesWith(
ConstantExpr::getGetElementPtr(NewGlobal, Indices2, true));
NewGlobal->takeName(G);
G->eraseFromParent();
Initializers[i] = ConstantStruct::get(
GlobalStructTy,
ConstantExpr::getPointerCast(NewGlobal, IntptrTy),
ConstantInt::get(IntptrTy, SizeInBytes),
ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
ConstantExpr::getPointerCast(Name, IntptrTy),
NULL);
DEBUG(dbgs() << "NEW GLOBAL:\n" << *NewGlobal);
}
ArrayType *ArrayOfGlobalStructTy = ArrayType::get(GlobalStructTy, n);
GlobalVariable *AllGlobals = new GlobalVariable(
M, ArrayOfGlobalStructTy, false, GlobalVariable::PrivateLinkage,
ConstantArray::get(ArrayOfGlobalStructTy, Initializers), "");
Function *AsanRegisterGlobals = cast<Function>(M.getOrInsertFunction(
kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
AsanRegisterGlobals->setLinkage(Function::ExternalLinkage);
IRB.CreateCall2(AsanRegisterGlobals,
IRB.CreatePointerCast(AllGlobals, IntptrTy),
ConstantInt::get(IntptrTy, n));
// We also need to unregister globals at the end, e.g. when a shared library
// gets closed.
Function *AsanDtorFunction = Function::Create(
FunctionType::get(Type::getVoidTy(*C), false),
GlobalValue::InternalLinkage, kAsanModuleDtorName, &M);
BasicBlock *AsanDtorBB = BasicBlock::Create(*C, "", AsanDtorFunction);
IRBuilder<> IRB_Dtor(ReturnInst::Create(*C, AsanDtorBB));
Function *AsanUnregisterGlobals = cast<Function>(M.getOrInsertFunction(
kAsanUnregisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage);
IRB_Dtor.CreateCall2(AsanUnregisterGlobals,
IRB.CreatePointerCast(AllGlobals, IntptrTy),
ConstantInt::get(IntptrTy, n));
appendToGlobalDtors(M, AsanDtorFunction, kAsanCtorAndCtorPriority);
DEBUG(dbgs() << M);
return true;
}
void MetadataTransform(CallSite &CS, const TargetData *TD)
{
Instruction *I = CS.getInstruction();
Module *M = I->getParent()->getParent()->getParent();
LLVMContext &Ctx = I->getContext();
unsigned argIdx = 0;
if (CS.arg_size() < 2)
report_fatal_error(I, "_SYS_lti_metadata requires at least two args");
// Parse the 'key' parameter
ConstantInt *CI = dyn_cast<ConstantInt>(CS.getArgument(argIdx++));
if (!CI)
report_fatal_error(I, "Metadata key must be a constant integer.");
uint16_t key = CI->getZExtValue();
if (key != CI->getZExtValue())
report_fatal_error(I, "Metadata key argument is too large.");
// Parse the 'fmt' parameter
std::string fmt;
if (!GetConstantStringInfo(CS.getArgument(argIdx++), fmt))
report_fatal_error(I, "Metadata format must be a constant string.");
if (fmt.size() != CS.arg_size() - 2)
report_fatal_error(I, "Length of metadata format must match number of parameters");
if (fmt.size() == 0)
report_fatal_error(I, "Empty metadata values are not supported");
/*
* Parse every other parameter according to the format string
*/
SmallVector<Constant*, 8> Members;
unsigned align = 1;
for (std::string::iterator FI = fmt.begin(), FE = fmt.end();
FI != FE; ++FI, argIdx++) {
Constant *Arg = dyn_cast<Constant>(CS.getArgument(argIdx));
Constant *C;
if (!Arg)
report_fatal_error(I, "Metadata argument " + Twine(argIdx+1) + " is not constant");
/*
* First, non-integer types
*/
switch (*FI) {
case 's': {
std::string str;
if (!GetConstantStringInfo(Arg, str))
report_fatal_error(I, "Metadata formatter 's' requires a constant string");
Members.push_back(ConstantArray::get(Ctx, str, false));
continue;
}
}
/*
* Integer types
*/
if (Arg->getType()->isPointerTy())
Arg = ConstantExpr::getPointerCast(Arg, Type::getInt32Ty(Ctx));
if (!Arg->getType()->isIntegerTy())
report_fatal_error(I, "Metadata argument " + Twine(argIdx+1) +
" can't be converted to an integer type.");
switch (*FI) {
case 'b':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt8Ty(Ctx), true);
break;
case 'B':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt8Ty(Ctx), false);
break;
case 'h':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt16Ty(Ctx), true);
break;
case 'H':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt16Ty(Ctx), false);
break;
case 'i':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt32Ty(Ctx), true);
break;
case 'I':
C = ConstantExpr::getIntegerCast(Arg, Type::getInt32Ty(Ctx), false);
break;
default:
report_fatal_error(I, "Unsupported format character '" + Twine(*FI) + "' in metadata");
}
align = std::max(align, TD->getABITypeAlignment(C->getType()));
Members.push_back(C);
}
Constant *Struct = ConstantStruct::getAnon(Members);
/*
* Install this metadata item as a global variable in a special section
*/
GlobalVariable *GV = new GlobalVariable(*M, Struct->getType(),
true, GlobalValue::ExternalLinkage, Struct, "", 0, false);
GV->setAlignment(align);
GV->setName(SVMDecorations::META + Twine(key) + SVMDecorations::SEPARATOR);
GV->setSection(".metadata");
// Remove the original _SYS_lti_metadata() call
I->eraseFromParent();
}
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
GlobalVariable *NamePtr = Inc->getName();
auto It = ProfileDataMap.find(NamePtr);
PerFunctionProfileData PD;
if (It != ProfileDataMap.end()) {
if (It->second.RegionCounters)
return It->second.RegionCounters;
PD = It->second;
}
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
// only one copy of counters of the COMDAT function will be emitted after
// linking.
Function *Fn = Inc->getParent()->getParent();
Comdat *ProfileVarsComdat = nullptr;
if (Fn->hasComdat())
ProfileVarsComdat = getOrCreateProfileComdat(*M, Inc);
NamePtr->setSection(getNameSection());
NamePtr->setAlignment(1);
NamePtr->setComdat(ProfileVarsComdat);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
auto *CounterPtr =
new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
Constant::getNullValue(CounterTy),
getVarName(Inc, getInstrProfCountersVarPrefix()));
CounterPtr->setVisibility(NamePtr->getVisibility());
CounterPtr->setSection(getCountersSection());
CounterPtr->setAlignment(8);
CounterPtr->setComdat(ProfileVarsComdat);
// Create data variable.
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
auto *Int16Ty = Type::getInt16Ty(Ctx);
auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
Type *DataTypes[] = {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) ?
ConstantExpr::getBitCast(Fn, Int8PtrTy) :
ConstantPointerNull::get(Int8PtrTy);
Constant *Int16ArrayVals[IPVK_Last+1];
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
Constant *DataVals[] = {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
getVarName(Inc, getInstrProfDataVarPrefix()));
Data->setVisibility(NamePtr->getVisibility());
Data->setSection(getDataSection());
Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
Data->setComdat(ProfileVarsComdat);
PD.RegionCounters = CounterPtr;
PD.DataVar = Data;
ProfileDataMap[NamePtr] = PD;
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
return CounterPtr;
}
Module *llvm::CloneModule(const Module *M,
DenseMap<const Value*, Value*> &ValueMap) {
// First off, we need to create the new module...
Module *New = new Module(M->getModuleIdentifier());
New->setDataLayout(M->getDataLayout());
New->setTargetTriple(M->getTargetTriple());
New->setModuleInlineAsm(M->getModuleInlineAsm());
// Copy all of the type symbol table entries over.
const TypeSymbolTable &TST = M->getTypeSymbolTable();
for (TypeSymbolTable::const_iterator TI = TST.begin(), TE = TST.end();
TI != TE; ++TI)
New->addTypeName(TI->first, TI->second);
// Copy all of the dependent libraries over.
for (Module::lib_iterator I = M->lib_begin(), E = M->lib_end(); I != E; ++I)
New->addLibrary(*I);
// Loop over all of the global variables, making corresponding globals in the
// new module. Here we add them to the ValueMap and to the new Module. We
// don't worry about attributes or initializers, they will come later.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = new GlobalVariable(I->getType()->getElementType(),
false,
GlobalValue::ExternalLinkage, 0,
I->getName(), New);
GV->setAlignment(I->getAlignment());
ValueMap[I] = GV;
}
// Loop over the functions in the module, making external functions as before
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *NF =
Function::Create(cast<FunctionType>(I->getType()->getElementType()),
GlobalValue::ExternalLinkage, I->getName(), New);
NF->copyAttributesFrom(I);
ValueMap[I] = NF;
}
// Loop over the aliases in the module
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I)
ValueMap[I] = new GlobalAlias(I->getType(), GlobalAlias::ExternalLinkage,
I->getName(), NULL, New);
// Now that all of the things that global variable initializer can refer to
// have been created, loop through and copy the global variable referrers
// over... We also set the attributes on the global now.
//
for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
I != E; ++I) {
GlobalVariable *GV = cast<GlobalVariable>(ValueMap[I]);
if (I->hasInitializer())
GV->setInitializer(cast<Constant>(MapValue(I->getInitializer(),
ValueMap)));
GV->setLinkage(I->getLinkage());
GV->setThreadLocal(I->isThreadLocal());
GV->setConstant(I->isConstant());
}
// Similarly, copy over function bodies now...
//
for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
Function *F = cast<Function>(ValueMap[I]);
if (!I->isDeclaration()) {
Function::arg_iterator DestI = F->arg_begin();
for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
++J) {
DestI->setName(J->getName());
ValueMap[J] = DestI++;
}
std::vector<ReturnInst*> Returns; // Ignore returns cloned...
CloneFunctionInto(F, I, ValueMap, Returns);
}
F->setLinkage(I->getLinkage());
}
// And aliases
for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
I != E; ++I) {
GlobalAlias *GA = cast<GlobalAlias>(ValueMap[I]);
GA->setLinkage(I->getLinkage());
if (const Constant* C = I->getAliasee())
GA->setAliasee(cast<Constant>(MapValue(C, ValueMap)));
}
return New;
}