/// computeTypeMapping - Loop over all of the linked values to compute type
/// mappings. For example, if we link "extern Foo *x" and "Foo *x = NULL", then
/// we have two struct types 'Foo' but one got renamed when the module was
/// loaded into the same LLVMContext.
void ModuleLinker::computeTypeMapping() {
// Incorporate globals.
for (Module::global_iterator I = SrcM->global_begin(),
E = SrcM->global_end(); I != E; ++I) {
GlobalValue *DGV = getLinkedToGlobal(I);
if (DGV == 0) continue;
if (!DGV->hasAppendingLinkage() || !I->hasAppendingLinkage()) {
TypeMap.addTypeMapping(DGV->getType(), I->getType());
continue;
}
// Unify the element type of appending arrays.
ArrayType *DAT = cast<ArrayType>(DGV->getType()->getElementType());
ArrayType *SAT = cast<ArrayType>(I->getType()->getElementType());
TypeMap.addTypeMapping(DAT->getElementType(), SAT->getElementType());
}
// Incorporate functions.
for (Module::iterator I = SrcM->begin(), E = SrcM->end(); I != E; ++I) {
if (GlobalValue *DGV = getLinkedToGlobal(I))
TypeMap.addTypeMapping(DGV->getType(), I->getType());
}
// Incorporate types by name, scanning all the types in the source module.
// At this point, the destination module may have a type "%foo = { i32 }" for
// example. When the source module got loaded into the same LLVMContext, if
// it had the same type, it would have been renamed to "%foo.42 = { i32 }".
// Though it isn't required for correctness, attempt to link these up to clean
// up the IR.
std::vector<StructType*> SrcStructTypes;
SrcM->findUsedStructTypes(SrcStructTypes);
SmallPtrSet<StructType*, 32> SrcStructTypesSet(SrcStructTypes.begin(),
SrcStructTypes.end());
for (unsigned i = 0, e = SrcStructTypes.size(); i != e; ++i) {
StructType *ST = SrcStructTypes[i];
if (!ST->hasName()) continue;
// Check to see if there is a dot in the name followed by a digit.
size_t DotPos = ST->getName().rfind('.');
if (DotPos == 0 || DotPos == StringRef::npos ||
ST->getName().back() == '.' || !isdigit(ST->getName()[DotPos+1]))
continue;
// Check to see if the destination module has a struct with the prefix name.
if (StructType *DST = DstM->getTypeByName(ST->getName().substr(0, DotPos)))
// Don't use it if this actually came from the source module. They're in
// the same LLVMContext after all.
if (!SrcStructTypesSet.count(DST))
TypeMap.addTypeMapping(DST, ST);
}
// Don't bother incorporating aliases, they aren't generally typed well.
// Now that we have discovered all of the type equivalences, get a body for
// any 'opaque' types in the dest module that are now resolved.
TypeMap.linkDefinedTypeBodies();
}
Constant *DevirtModule::getPointerAtOffset(Constant *I, uint64_t Offset) {
if (I->getType()->isPointerTy()) {
if (Offset == 0)
return I;
return nullptr;
}
const DataLayout &DL = M.getDataLayout();
if (auto *C = dyn_cast<ConstantStruct>(I)) {
const StructLayout *SL = DL.getStructLayout(C->getType());
if (Offset >= SL->getSizeInBytes())
return nullptr;
unsigned Op = SL->getElementContainingOffset(Offset);
return getPointerAtOffset(cast<Constant>(I->getOperand(Op)),
Offset - SL->getElementOffset(Op));
}
if (auto *C = dyn_cast<ConstantArray>(I)) {
ArrayType *VTableTy = C->getType();
uint64_t ElemSize = DL.getTypeAllocSize(VTableTy->getElementType());
unsigned Op = Offset / ElemSize;
if (Op >= C->getNumOperands())
return nullptr;
return getPointerAtOffset(cast<Constant>(I->getOperand(Op)),
Offset % ElemSize);
}
return nullptr;
}
/* This routine extracts the filename of the file input to the open call
This function has been borrowed from the LLPE toolchain by Smowton.
*/
bool getConstantStringInfo(const Value *V, StringRef &Str) {
// Look through bitcast instructions and geps.
V = V->stripPointerCasts();
// If the value is a GEP instruction or constant expression, treat it as an offset.
if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
// Make sure the GEP has exactly three arguments.
if (GEP->getNumOperands() != 3)
return false;
// Make sure the index-ee is a pointer to array of i8.
PointerType *PT = cast<PointerType>(GEP->getOperand(0)->getType());
ArrayType *AT = dyn_cast<ArrayType>(PT->getElementType());
if (AT == 0 || !AT->getElementType()->isIntegerTy(8))
return false;
// Check to make sure that the first operand of the GEP is an integer and
// has value 0 so that we are sure we're indexing into the initializer.
const ConstantInt *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
if (FirstIdx == 0 || !FirstIdx->isZero())
return false;
// If the second index isn't a ConstantInt, then this is a variable index
// into the array. If this occurs, we can't say anything meaningful about
// the string.
uint64_t StartIdx = 0;
if (const ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(2)))
StartIdx = CI->getZExtValue();
else
return false;
}
// The GEP instruction, constant or instruction, must reference a global
// variable that is a constant and is initialized. The referenced constant
// initializer is the array that we'll use for optimization.
const GlobalVariable *GV = dyn_cast<GlobalVariable>(V);
if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
return false;
// Handle the all-zeros case
if (GV->getInitializer()->isNullValue()) {
// This is a degenerate case. The initializer is constant zero so the
// length of the string must be zero.
Str = "";
return true;
}
// Must be a Constant Array
const ConstantDataArray *Array =
dyn_cast<ConstantDataArray>(GV->getInitializer());
if (Array == 0 || !Array->isString())
return false;
// Start out with the entire array in the StringRef.
Str = Array->getAsString();
return true;
}
AllocaInst* Variables::changeLocal(Value* value, ArrayType* newType) {
AllocaInst* oldTarget = dyn_cast<AllocaInst>(value);
PointerType* oldPointerType = dyn_cast<PointerType>(oldTarget->getType());
ArrayType* oldType = dyn_cast<ArrayType>(oldPointerType->getElementType());
AllocaInst* newTarget = NULL;
errs() << "Changing the precision of variable \"" << oldTarget->getName() << "\" from " << *oldType
<< " to " << *newType << ".\n";
if (newType->getElementType()->getTypeID() != oldType->getElementType()->getTypeID()) {
newTarget = new AllocaInst(newType, getInt32(1), "", oldTarget);
// we are not calling getAlignment because in this case double requires 16. Investigate further.
unsigned alignment;
switch(newType->getElementType()->getTypeID()) {
case Type::FloatTyID:
alignment = 4;
break;
case Type::DoubleTyID:
alignment = 16;
break;
case Type::X86_FP80TyID:
alignment = 16;
break;
default:
alignment = 0;
}
newTarget->setAlignment(alignment); // depends on type? 8 for float? 16 for double?
newTarget->takeName(oldTarget);
// iterating through instructions using old AllocaInst
vector<Instruction*> erase;
Value::use_iterator it = oldTarget->use_begin();
for(; it != oldTarget->use_end(); it++) {
bool is_erased = Transformer::transform(it, newTarget, oldTarget, newType, oldType, alignment);
if (!is_erased)
erase.push_back(dyn_cast<Instruction>(*it));
}
// erasing uses of old instructions
for(unsigned int i = 0; i < erase.size(); i++) {
erase[i]->eraseFromParent();
}
// erase old instruction
//oldTarget->eraseFromParent();
}
else {
errs() << "\tNo changes required.\n";
}
return newTarget;
}
bool EfficiencySanitizer::instrumentGetElementPtr(Instruction *I, Module &M) {
GetElementPtrInst *GepInst = dyn_cast<GetElementPtrInst>(I);
bool Res = false;
if (GepInst == nullptr || GepInst->getNumIndices() == 1) {
++NumIgnoredGEPs;
return false;
}
Type *SourceTy = GepInst->getSourceElementType();
StructType *StructTy = nullptr;
ConstantInt *Idx;
// Check if GEP calculates address from a struct array.
if (isa<StructType>(SourceTy)) {
StructTy = cast<StructType>(SourceTy);
Idx = dyn_cast<ConstantInt>(GepInst->getOperand(1));
if ((Idx == nullptr || Idx->getSExtValue() != 0) &&
!shouldIgnoreStructType(StructTy) && StructTyMap.count(StructTy) != 0)
Res |= insertCounterUpdate(I, StructTy, getArrayCounterIdx(StructTy));
}
// Iterate all (except the first and the last) idx within each GEP instruction
// for possible nested struct field address calculation.
for (unsigned i = 1; i < GepInst->getNumIndices(); ++i) {
SmallVector<Value *, 8> IdxVec(GepInst->idx_begin(),
GepInst->idx_begin() + i);
Type *Ty = GetElementPtrInst::getIndexedType(SourceTy, IdxVec);
unsigned CounterIdx = 0;
if (isa<ArrayType>(Ty)) {
ArrayType *ArrayTy = cast<ArrayType>(Ty);
StructTy = dyn_cast<StructType>(ArrayTy->getElementType());
if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0)
continue;
// The last counter for struct array access.
CounterIdx = getArrayCounterIdx(StructTy);
} else if (isa<StructType>(Ty)) {
StructTy = cast<StructType>(Ty);
if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0)
continue;
// Get the StructTy's subfield index.
Idx = cast<ConstantInt>(GepInst->getOperand(i+1));
assert(Idx->getSExtValue() >= 0 &&
Idx->getSExtValue() < StructTy->getNumElements());
CounterIdx = getFieldCounterIdx(StructTy) + Idx->getSExtValue();
}
Res |= insertCounterUpdate(I, StructTy, CounterIdx);
}
if (Res)
++NumInstrumentedGEPs;
else
++NumIgnoredGEPs;
return Res;
}
static void appendToGlobalArray(const char *Array,
Module &M, Function *F, int Priority) {
IRBuilder<> IRB(M.getContext());
FunctionType *FnTy = FunctionType::get(IRB.getVoidTy(), false);
// Get the current set of static global constructors and add the new ctor
// to the list.
SmallVector<Constant *, 16> CurrentCtors;
StructType *EltTy;
if (GlobalVariable *GVCtor = M.getNamedGlobal(Array)) {
// If there is a global_ctors array, use the existing struct type, which can
// have 2 or 3 fields.
ArrayType *ATy = cast<ArrayType>(GVCtor->getType()->getElementType());
EltTy = cast<StructType>(ATy->getElementType());
if (Constant *Init = GVCtor->getInitializer()) {
unsigned n = Init->getNumOperands();
CurrentCtors.reserve(n + 1);
for (unsigned i = 0; i != n; ++i)
CurrentCtors.push_back(cast<Constant>(Init->getOperand(i)));
}
GVCtor->eraseFromParent();
} else {
// Use a simple two-field struct if there isn't one already.
EltTy = StructType::get(IRB.getInt32Ty(), PointerType::getUnqual(FnTy),
nullptr);
}
// Build a 2 or 3 field global_ctor entry. We don't take a comdat key.
Constant *CSVals[3];
CSVals[0] = IRB.getInt32(Priority);
CSVals[1] = F;
// FIXME: Drop support for the two element form in LLVM 4.0.
if (EltTy->getNumElements() >= 3)
CSVals[2] = llvm::Constant::getNullValue(IRB.getInt8PtrTy());
Constant *RuntimeCtorInit =
ConstantStruct::get(EltTy, makeArrayRef(CSVals, EltTy->getNumElements()));
CurrentCtors.push_back(RuntimeCtorInit);
// Create a new initializer.
ArrayType *AT = ArrayType::get(EltTy, CurrentCtors.size());
Constant *NewInit = ConstantArray::get(AT, CurrentCtors);
// Create the new global variable and replace all uses of
// the old global variable with the new one.
(void)new GlobalVariable(M, NewInit->getType(), false,
GlobalValue::AppendingLinkage, NewInit, Array);
}
ConstantInt* Variables::getAlignmentInBits(Type *type) {
ConstantInt *alignment = NULL;
ArrayType *arrayType = dyn_cast<ArrayType>(type);
switch (arrayType->getElementType()->getTypeID()) {
case Type::FloatTyID:
alignment = getInt64(32);
break;
case Type::DoubleTyID:
alignment = getInt64(64);
break;
default:
errs() << "WARNING: Unhandled type @ getAlignmentInBits\n";
exit(1);
}
return alignment;
}
ConstantInt* Variables::getSizeInBits(Type *type) {
ConstantInt *size = NULL;
ArrayType *arrayType = dyn_cast<ArrayType>(type);
switch (arrayType->getElementType()->getTypeID()) {
case Type::FloatTyID: {
int isize = arrayType->getNumElements() * 32;
size = getInt64(isize);
break;
}
case Type::DoubleTyID: {
int isize = arrayType->getNumElements() * 64;
size = getInt64(isize);
break;
}
default:
errs() << "WARNING: Unhandled type @ getSizeInBits\n";
exit(1);
}
return size;
}
bool DevirtModule::tryFindVirtualCallTargets(
std::vector<VirtualCallTarget> &TargetsForSlot,
const std::set<BitSetInfo> &BitSetInfos, uint64_t ByteOffset) {
for (const BitSetInfo &BS : BitSetInfos) {
if (!BS.Bits->GV->isConstant())
return false;
auto Init = dyn_cast<ConstantArray>(BS.Bits->GV->getInitializer());
if (!Init)
return false;
ArrayType *VTableTy = Init->getType();
uint64_t ElemSize =
M.getDataLayout().getTypeAllocSize(VTableTy->getElementType());
uint64_t GlobalSlotOffset = BS.Offset + ByteOffset;
if (GlobalSlotOffset % ElemSize != 0)
return false;
unsigned Op = GlobalSlotOffset / ElemSize;
if (Op >= Init->getNumOperands())
return false;
auto Fn = dyn_cast<Function>(Init->getOperand(Op)->stripPointerCasts());
if (!Fn)
return false;
// We can disregard __cxa_pure_virtual as a possible call target, as
// calls to pure virtuals are UB.
if (Fn->getName() == "__cxa_pure_virtual")
continue;
TargetsForSlot.push_back({Fn, &BS});
}
// Give up if we couldn't find any targets.
return !TargetsForSlot.empty();
}
uint64_t TargetData::getTypeSizeInBits(Type *Ty) const {
assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
switch (Ty->getTypeID()) {
case Type::LabelTyID:
case Type::PointerTyID:
return getPointerSizeInBits();
case Type::ArrayTyID: {
ArrayType *ATy = cast<ArrayType>(Ty);
return getTypeAllocSizeInBits(ATy->getElementType())*ATy->getNumElements();
}
case Type::StructTyID:
// Get the layout annotation... which is lazily created on demand.
return getStructLayout(cast<StructType>(Ty))->getSizeInBits();
case Type::IntegerTyID:
return cast<IntegerType>(Ty)->getBitWidth();
case Type::VoidTyID:
return 8;
case Type::HalfTyID:
return 16;
case Type::FloatTyID:
return 32;
case Type::DoubleTyID:
case Type::X86_MMXTyID:
return 64;
case Type::PPC_FP128TyID:
case Type::FP128TyID:
return 128;
// In memory objects this is always aligned to a higher boundary, but
// only 80 bits contain information.
case Type::X86_FP80TyID:
return 80;
case Type::VectorTyID:
return cast<VectorType>(Ty)->getBitWidth();
default:
llvm_unreachable("TargetData::getTypeSizeInBits(): Unsupported type");
}
}
static bool tryPromoteAllocaToVector(AllocaInst *Alloca, AMDGPUAS AS) {
ArrayType *AllocaTy = dyn_cast<ArrayType>(Alloca->getAllocatedType());
DEBUG(dbgs() << "Alloca candidate for vectorization\n");
// FIXME: There is no reason why we can't support larger arrays, we
// are just being conservative for now.
// FIXME: We also reject alloca's of the form [ 2 x [ 2 x i32 ]] or equivalent. Potentially these
// could also be promoted but we don't currently handle this case
if (!AllocaTy ||
AllocaTy->getNumElements() > 4 ||
AllocaTy->getNumElements() < 2 ||
!VectorType::isValidElementType(AllocaTy->getElementType())) {
DEBUG(dbgs() << " Cannot convert type to vector\n");
return false;
}
std::map<GetElementPtrInst*, Value*> GEPVectorIdx;
std::vector<Value*> WorkList;
for (User *AllocaUser : Alloca->users()) {
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
if (!GEP) {
if (!canVectorizeInst(cast<Instruction>(AllocaUser), Alloca))
return false;
WorkList.push_back(AllocaUser);
continue;
}
Value *Index = GEPToVectorIndex(GEP);
// If we can't compute a vector index from this GEP, then we can't
// promote this alloca to vector.
if (!Index) {
DEBUG(dbgs() << " Cannot compute vector index for GEP " << *GEP << '\n');
return false;
}
GEPVectorIdx[GEP] = Index;
for (User *GEPUser : AllocaUser->users()) {
if (!canVectorizeInst(cast<Instruction>(GEPUser), AllocaUser))
return false;
WorkList.push_back(GEPUser);
}
}
VectorType *VectorTy = arrayTypeToVecType(AllocaTy);
DEBUG(dbgs() << " Converting alloca to vector "
<< *AllocaTy << " -> " << *VectorTy << '\n');
for (Value *V : WorkList) {
Instruction *Inst = cast<Instruction>(V);
IRBuilder<> Builder(Inst);
switch (Inst->getOpcode()) {
case Instruction::Load: {
Type *VecPtrTy = VectorTy->getPointerTo(AS.PRIVATE_ADDRESS);
Value *Ptr = cast<LoadInst>(Inst)->getPointerOperand();
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
Value *VecValue = Builder.CreateLoad(BitCast);
Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
Inst->replaceAllUsesWith(ExtractElement);
Inst->eraseFromParent();
break;
}
case Instruction::Store: {
Type *VecPtrTy = VectorTy->getPointerTo(AS.PRIVATE_ADDRESS);
StoreInst *SI = cast<StoreInst>(Inst);
Value *Ptr = SI->getPointerOperand();
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
Value *VecValue = Builder.CreateLoad(BitCast);
Value *NewVecValue = Builder.CreateInsertElement(VecValue,
SI->getValueOperand(),
Index);
Builder.CreateStore(NewVecValue, BitCast);
Inst->eraseFromParent();
break;
}
case Instruction::BitCast:
case Instruction::AddrSpaceCast:
break;
default:
llvm_unreachable("Inconsistency in instructions promotable to vector");
}
}
return true;
}
/// If there were any appending global variables, link them together now.
/// Return true on error.
Constant *IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,
const GlobalVariable *SrcGV) {
Type *EltTy = cast<ArrayType>(TypeMap.get(SrcGV->getType()->getElementType()))
->getElementType();
StringRef Name = SrcGV->getName();
bool IsNewStructor = false;
bool IsOldStructor = false;
if (Name == "llvm.global_ctors" || Name == "llvm.global_dtors") {
if (cast<StructType>(EltTy)->getNumElements() == 3)
IsNewStructor = true;
else
IsOldStructor = true;
}
PointerType *VoidPtrTy = Type::getInt8Ty(SrcGV->getContext())->getPointerTo();
if (IsOldStructor) {
auto &ST = *cast<StructType>(EltTy);
Type *Tys[3] = {ST.getElementType(0), ST.getElementType(1), VoidPtrTy};
EltTy = StructType::get(SrcGV->getContext(), Tys, false);
}
if (DstGV) {
ArrayType *DstTy = cast<ArrayType>(DstGV->getType()->getElementType());
if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage()) {
emitError(
"Linking globals named '" + SrcGV->getName() +
"': can only link appending global with another appending global!");
return nullptr;
}
// Check to see that they two arrays agree on type.
if (EltTy != DstTy->getElementType()) {
emitError("Appending variables with different element types!");
return nullptr;
}
if (DstGV->isConstant() != SrcGV->isConstant()) {
emitError("Appending variables linked with different const'ness!");
return nullptr;
}
if (DstGV->getAlignment() != SrcGV->getAlignment()) {
emitError(
"Appending variables with different alignment need to be linked!");
return nullptr;
}
if (DstGV->getVisibility() != SrcGV->getVisibility()) {
emitError(
"Appending variables with different visibility need to be linked!");
return nullptr;
}
if (DstGV->hasUnnamedAddr() != SrcGV->hasUnnamedAddr()) {
emitError(
"Appending variables with different unnamed_addr need to be linked!");
return nullptr;
}
if (StringRef(DstGV->getSection()) != SrcGV->getSection()) {
emitError(
"Appending variables with different section name need to be linked!");
return nullptr;
}
}
SmallVector<Constant *, 16> DstElements;
if (DstGV)
getArrayElements(DstGV->getInitializer(), DstElements);
SmallVector<Constant *, 16> SrcElements;
getArrayElements(SrcGV->getInitializer(), SrcElements);
if (IsNewStructor)
SrcElements.erase(
std::remove_if(SrcElements.begin(), SrcElements.end(),
[this](Constant *E) {
auto *Key = dyn_cast<GlobalValue>(
E->getAggregateElement(2)->stripPointerCasts());
if (!Key)
return false;
GlobalValue *DGV = getLinkedToGlobal(Key);
return !shouldLink(DGV, *Key);
}),
SrcElements.end());
uint64_t NewSize = DstElements.size() + SrcElements.size();
ArrayType *NewType = ArrayType::get(EltTy, NewSize);
// Create the new global variable.
GlobalVariable *NG = new GlobalVariable(
DstM, NewType, SrcGV->isConstant(), SrcGV->getLinkage(),
/*init*/ nullptr, /*name*/ "", DstGV, SrcGV->getThreadLocalMode(),
SrcGV->getType()->getAddressSpace());
NG->copyAttributesFrom(SrcGV);
forceRenaming(NG, SrcGV->getName());
Constant *Ret = ConstantExpr::getBitCast(NG, TypeMap.get(SrcGV->getType()));
// Stop recursion.
ValueMap[SrcGV] = Ret;
for (auto *V : SrcElements) {
Constant *NewV;
if (IsOldStructor) {
auto *S = cast<ConstantStruct>(V);
auto *E1 = MapValue(S->getOperand(0), ValueMap, RF_MoveDistinctMDs,
&TypeMap, &GValMaterializer);
auto *E2 = MapValue(S->getOperand(1), ValueMap, RF_MoveDistinctMDs,
&TypeMap, &GValMaterializer);
Value *Null = Constant::getNullValue(VoidPtrTy);
NewV =
ConstantStruct::get(cast<StructType>(EltTy), E1, E2, Null, nullptr);
} else {
NewV = MapValue(V, ValueMap, RF_MoveDistinctMDs, &TypeMap,
&GValMaterializer);
}
DstElements.push_back(NewV);
}
NG->setInitializer(ConstantArray::get(NewType, DstElements));
// Replace any uses of the two global variables with uses of the new
// global.
if (DstGV) {
DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
DstGV->eraseFromParent();
}
return Ret;
}
/// linkAppendingVarProto - If there were any appending global variables, link
/// them together now. Return true on error.
bool ModuleLinker::linkAppendingVarProto(GlobalVariable *DstGV,
GlobalVariable *SrcGV) {
if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage())
return emitError("Linking globals named '" + SrcGV->getName() +
"': can only link appending global with another appending global!");
ArrayType *DstTy = cast<ArrayType>(DstGV->getType()->getElementType());
ArrayType *SrcTy =
cast<ArrayType>(TypeMap.get(SrcGV->getType()->getElementType()));
Type *EltTy = DstTy->getElementType();
// Check to see that they two arrays agree on type.
if (EltTy != SrcTy->getElementType())
return emitError("Appending variables with different element types!");
if (DstGV->isConstant() != SrcGV->isConstant())
return emitError("Appending variables linked with different const'ness!");
if (DstGV->getAlignment() != SrcGV->getAlignment())
return emitError(
"Appending variables with different alignment need to be linked!");
if (DstGV->getVisibility() != SrcGV->getVisibility())
return emitError(
"Appending variables with different visibility need to be linked!");
if (DstGV->getSection() != SrcGV->getSection())
return emitError(
"Appending variables with different section name need to be linked!");
uint64_t NewSize = DstTy->getNumElements() + SrcTy->getNumElements();
ArrayType *NewType = ArrayType::get(EltTy, NewSize);
// Create the new global variable.
GlobalVariable *NG =
new GlobalVariable(*DstGV->getParent(), NewType, SrcGV->isConstant(),
DstGV->getLinkage(), /*init*/0, /*name*/"", DstGV,
DstGV->isThreadLocal(),
DstGV->getType()->getAddressSpace());
// Propagate alignment, visibility and section info.
CopyGVAttributes(NG, DstGV);
AppendingVarInfo AVI;
AVI.NewGV = NG;
AVI.DstInit = DstGV->getInitializer();
AVI.SrcInit = SrcGV->getInitializer();
AppendingVars.push_back(AVI);
// Replace any uses of the two global variables with uses of the new
// global.
ValueMap[SrcGV] = ConstantExpr::getBitCast(NG, TypeMap.get(SrcGV->getType()));
DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
DstGV->eraseFromParent();
// Track the source variable so we don't try to link it.
DoNotLinkFromSource.insert(SrcGV);
return false;
}
static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt)
{
// assumes `jl_isbits(bt)`.
// `ptr` can point to a inline field, do not read the tag from it.
// make sure to return exactly the type specified by
// julia_type_to_llvm as this will be assumed by the callee.
if (bt == jl_bool_type)
return ConstantInt::get(T_int8, (*(const uint8_t*)ptr) ? 1 : 0);
if (jl_is_vecelement_type((jl_value_t*)bt))
bt = (jl_datatype_t*)jl_tparam0(bt);
Type *lt = julia_struct_to_llvm((jl_value_t*)bt, NULL, NULL);
if (type_is_ghost(lt))
return UndefValue::get(NoopType);
if (jl_is_primitivetype(bt)) {
if (lt->isFloatTy()) {
uint32_t data32 = *(const uint32_t*)ptr;
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), APInt(32, data32)));
}
if (lt->isDoubleTy()) {
uint64_t data64 = *(const uint64_t*)ptr;
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), APInt(64, data64)));
}
int nb = jl_datatype_size(bt);
APInt val(8 * nb, 0);
void *bits = const_cast<uint64_t*>(val.getRawData());
assert(sys::IsLittleEndianHost);
memcpy(bits, ptr, nb);
if (lt->isFloatingPointTy()) {
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), val));
}
assert(cast<IntegerType>(lt)->getBitWidth() == 8u * nb);
return ConstantInt::get(lt, val);
}
CompositeType *lct = cast<CompositeType>(lt);
size_t nf = jl_datatype_nfields(bt);
std::vector<Constant*> fields(nf);
for (size_t i = 0; i < nf; i++) {
size_t offs = jl_field_offset(bt, i);
assert(!jl_field_isptr(bt, i));
jl_value_t *ft = jl_field_type(bt, i);
Type *lft = lct->getTypeAtIndex(i);
const uint8_t *ov = (const uint8_t*)ptr + offs;
Constant *val;
if (jl_is_uniontype(ft)) {
// compute the same type layout as julia_struct_to_llvm
size_t fsz = jl_field_size(bt, i);
size_t al = jl_field_align(bt, i);
uint8_t sel = ((const uint8_t*)ptr)[offs + fsz - 1];
jl_value_t *active_ty = jl_nth_union_component(ft, sel);
size_t active_sz = jl_datatype_size(active_ty);
ArrayType *aty = cast<ArrayType>(cast<StructType>(lft)->getTypeAtIndex(0u));
assert(aty->getElementType() == IntegerType::get(jl_LLVMContext, 8 * al) &&
aty->getNumElements() == (fsz - 1) / al);
std::vector<Constant*> ArrayElements(0);
for (unsigned j = 0; j < aty->getNumElements(); j++) {
APInt Elem(8 * al, 0);
void *bits = const_cast<uint64_t*>(Elem.getRawData());
if (active_sz > al) {
memcpy(bits, ov, al);
active_sz -= al;
}
else if (active_sz > 0) {
memcpy(bits, ov, active_sz);
active_sz = 0;
}
ov += al;
ArrayElements.push_back(ConstantInt::get(aty->getElementType(), Elem));
}
std::vector<Constant*> Elements(0);
Elements.push_back(ConstantArray::get(aty, ArrayElements));
unsigned remainder = (fsz - 1) % al;
while (remainder--) {
uint8_t byte;
if (active_sz > 0) {
byte = *ov;
active_sz -= 1;
}
else {
byte = 0;
}
ov += 1;
APInt Elem(8, byte);
Elements.push_back(ConstantInt::get(T_int8, Elem));
}
Elements.push_back(ConstantInt::get(T_int8, sel));
val = ConstantStruct::get(cast<StructType>(lft), Elements);
}
else {
val = julia_const_to_llvm(ov, (jl_datatype_t*)ft);
}
fields[i] = val;
}
if (lct->isVectorTy())
return ConstantVector::get(fields);
if (StructType *st = dyn_cast<StructType>(lct))
return ConstantStruct::get(st, fields);
ArrayType *at = cast<ArrayType>(lct);
return ConstantArray::get(at, fields);
}
/// Loop over all of the linked values to compute type mappings. For example,
/// if we link "extern Foo *x" and "Foo *x = NULL", then we have two struct
/// types 'Foo' but one got renamed when the module was loaded into the same
/// LLVMContext.
void IRLinker::computeTypeMapping() {
for (GlobalValue &SGV : SrcM->globals()) {
GlobalValue *DGV = getLinkedToGlobal(&SGV);
if (!DGV)
continue;
if (!DGV->hasAppendingLinkage() || !SGV.hasAppendingLinkage()) {
TypeMap.addTypeMapping(DGV->getType(), SGV.getType());
continue;
}
// Unify the element type of appending arrays.
ArrayType *DAT = cast<ArrayType>(DGV->getValueType());
ArrayType *SAT = cast<ArrayType>(SGV.getValueType());
TypeMap.addTypeMapping(DAT->getElementType(), SAT->getElementType());
}
for (GlobalValue &SGV : *SrcM)
if (GlobalValue *DGV = getLinkedToGlobal(&SGV))
TypeMap.addTypeMapping(DGV->getType(), SGV.getType());
for (GlobalValue &SGV : SrcM->aliases())
if (GlobalValue *DGV = getLinkedToGlobal(&SGV))
TypeMap.addTypeMapping(DGV->getType(), SGV.getType());
// Incorporate types by name, scanning all the types in the source module.
// At this point, the destination module may have a type "%foo = { i32 }" for
// example. When the source module got loaded into the same LLVMContext, if
// it had the same type, it would have been renamed to "%foo.42 = { i32 }".
std::vector<StructType *> Types = SrcM->getIdentifiedStructTypes();
for (StructType *ST : Types) {
if (!ST->hasName())
continue;
// Check to see if there is a dot in the name followed by a digit.
size_t DotPos = ST->getName().rfind('.');
if (DotPos == 0 || DotPos == StringRef::npos ||
ST->getName().back() == '.' ||
!isdigit(static_cast<unsigned char>(ST->getName()[DotPos + 1])))
continue;
// Check to see if the destination module has a struct with the prefix name.
StructType *DST = DstM.getTypeByName(ST->getName().substr(0, DotPos));
if (!DST)
continue;
// Don't use it if this actually came from the source module. They're in
// the same LLVMContext after all. Also don't use it unless the type is
// actually used in the destination module. This can happen in situations
// like this:
//
// Module A Module B
// -------- --------
// %Z = type { %A } %B = type { %C.1 }
// %A = type { %B.1, [7 x i8] } %C.1 = type { i8* }
// %B.1 = type { %C } %A.2 = type { %B.3, [5 x i8] }
// %C = type { i8* } %B.3 = type { %C.1 }
//
// When we link Module B with Module A, the '%B' in Module B is
// used. However, that would then use '%C.1'. But when we process '%C.1',
// we prefer to take the '%C' version. So we are then left with both
// '%C.1' and '%C' being used for the same types. This leads to some
// variables using one type and some using the other.
if (TypeMap.DstStructTypesSet.hasType(DST))
TypeMap.addTypeMapping(DST, ST);
}
// Now that we have discovered all of the type equivalences, get a body for
// any 'opaque' types in the dest module that are now resolved.
TypeMap.linkDefinedTypeBodies();
}
static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
ArrayType *AllocaTy = dyn_cast<ArrayType>(Alloca->getAllocatedType());
DEBUG(dbgs() << "Alloca candidate for vectorization\n");
// FIXME: There is no reason why we can't support larger arrays, we
// are just being conservative for now.
if (!AllocaTy ||
AllocaTy->getElementType()->isVectorTy() ||
AllocaTy->getNumElements() > 4) {
DEBUG(dbgs() << " Cannot convert type to vector\n");
return false;
}
std::map<GetElementPtrInst*, Value*> GEPVectorIdx;
std::vector<Value*> WorkList;
for (User *AllocaUser : Alloca->users()) {
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
if (!GEP) {
if (!canVectorizeInst(cast<Instruction>(AllocaUser), Alloca))
return false;
WorkList.push_back(AllocaUser);
continue;
}
Value *Index = GEPToVectorIndex(GEP);
// If we can't compute a vector index from this GEP, then we can't
// promote this alloca to vector.
if (!Index) {
DEBUG(dbgs() << " Cannot compute vector index for GEP " << *GEP << '\n');
return false;
}
GEPVectorIdx[GEP] = Index;
for (User *GEPUser : AllocaUser->users()) {
if (!canVectorizeInst(cast<Instruction>(GEPUser), AllocaUser))
return false;
WorkList.push_back(GEPUser);
}
}
VectorType *VectorTy = arrayTypeToVecType(AllocaTy);
DEBUG(dbgs() << " Converting alloca to vector "
<< *AllocaTy << " -> " << *VectorTy << '\n');
for (Value *V : WorkList) {
Instruction *Inst = cast<Instruction>(V);
IRBuilder<> Builder(Inst);
switch (Inst->getOpcode()) {
case Instruction::Load: {
Value *Ptr = Inst->getOperand(0);
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
Value *VecValue = Builder.CreateLoad(BitCast);
Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
Inst->replaceAllUsesWith(ExtractElement);
Inst->eraseFromParent();
break;
}
case Instruction::Store: {
Value *Ptr = Inst->getOperand(1);
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
Value *VecValue = Builder.CreateLoad(BitCast);
Value *NewVecValue = Builder.CreateInsertElement(VecValue,
Inst->getOperand(0),
Index);
Builder.CreateStore(NewVecValue, BitCast);
Inst->eraseFromParent();
break;
}
case Instruction::BitCast:
case Instruction::AddrSpaceCast:
break;
default:
Inst->dump();
llvm_unreachable("Inconsistency in instructions promotable to vector");
}
}
return true;
}
/// If there were any appending global variables, link them together now.
Expected<Constant *>
IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,
const GlobalVariable *SrcGV) {
Type *EltTy = cast<ArrayType>(TypeMap.get(SrcGV->getValueType()))
->getElementType();
// FIXME: This upgrade is done during linking to support the C API. Once the
// old form is deprecated, we should move this upgrade to
// llvm::UpgradeGlobalVariable() and simplify the logic here and in
// Mapper::mapAppendingVariable() in ValueMapper.cpp.
StringRef Name = SrcGV->getName();
bool IsNewStructor = false;
bool IsOldStructor = false;
if (Name == "llvm.global_ctors" || Name == "llvm.global_dtors") {
if (cast<StructType>(EltTy)->getNumElements() == 3)
IsNewStructor = true;
else
IsOldStructor = true;
}
PointerType *VoidPtrTy = Type::getInt8Ty(SrcGV->getContext())->getPointerTo();
if (IsOldStructor) {
auto &ST = *cast<StructType>(EltTy);
Type *Tys[3] = {ST.getElementType(0), ST.getElementType(1), VoidPtrTy};
EltTy = StructType::get(SrcGV->getContext(), Tys, false);
}
uint64_t DstNumElements = 0;
if (DstGV) {
ArrayType *DstTy = cast<ArrayType>(DstGV->getValueType());
DstNumElements = DstTy->getNumElements();
if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage())
return stringErr(
"Linking globals named '" + SrcGV->getName() +
"': can only link appending global with another appending "
"global!");
// Check to see that they two arrays agree on type.
if (EltTy != DstTy->getElementType())
return stringErr("Appending variables with different element types!");
if (DstGV->isConstant() != SrcGV->isConstant())
return stringErr("Appending variables linked with different const'ness!");
if (DstGV->getAlignment() != SrcGV->getAlignment())
return stringErr(
"Appending variables with different alignment need to be linked!");
if (DstGV->getVisibility() != SrcGV->getVisibility())
return stringErr(
"Appending variables with different visibility need to be linked!");
if (DstGV->hasGlobalUnnamedAddr() != SrcGV->hasGlobalUnnamedAddr())
return stringErr(
"Appending variables with different unnamed_addr need to be linked!");
if (DstGV->getSection() != SrcGV->getSection())
return stringErr(
"Appending variables with different section name need to be linked!");
}
SmallVector<Constant *, 16> SrcElements;
getArrayElements(SrcGV->getInitializer(), SrcElements);
if (IsNewStructor) {
auto It = remove_if(SrcElements, [this](Constant *E) {
auto *Key =
dyn_cast<GlobalValue>(E->getAggregateElement(2)->stripPointerCasts());
if (!Key)
return false;
GlobalValue *DGV = getLinkedToGlobal(Key);
return !shouldLink(DGV, *Key);
});
SrcElements.erase(It, SrcElements.end());
}
uint64_t NewSize = DstNumElements + SrcElements.size();
ArrayType *NewType = ArrayType::get(EltTy, NewSize);
// Create the new global variable.
GlobalVariable *NG = new GlobalVariable(
DstM, NewType, SrcGV->isConstant(), SrcGV->getLinkage(),
/*init*/ nullptr, /*name*/ "", DstGV, SrcGV->getThreadLocalMode(),
SrcGV->getType()->getAddressSpace());
NG->copyAttributesFrom(SrcGV);
forceRenaming(NG, SrcGV->getName());
Constant *Ret = ConstantExpr::getBitCast(NG, TypeMap.get(SrcGV->getType()));
Mapper.scheduleMapAppendingVariable(*NG,
DstGV ? DstGV->getInitializer() : nullptr,
IsOldStructor, SrcElements);
// Replace any uses of the two global variables with uses of the new
// global.
if (DstGV) {
DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
DstGV->eraseFromParent();
}
return Ret;
}
/// cmpType - compares two types,
/// defines total ordering among the types set.
/// See method declaration comments for more details.
int FunctionComparator::cmpType(Type *TyL, Type *TyR) const {
PointerType *PTyL = dyn_cast<PointerType>(TyL);
PointerType *PTyR = dyn_cast<PointerType>(TyR);
if (DL) {
if (PTyL && PTyL->getAddressSpace() == 0) TyL = DL->getIntPtrType(TyL);
if (PTyR && PTyR->getAddressSpace() == 0) TyR = DL->getIntPtrType(TyR);
}
if (TyL == TyR)
return 0;
if (int Res = cmpNumbers(TyL->getTypeID(), TyR->getTypeID()))
return Res;
switch (TyL->getTypeID()) {
default:
llvm_unreachable("Unknown type!");
// Fall through in Release mode.
case Type::IntegerTyID:
case Type::VectorTyID:
// TyL == TyR would have returned true earlier.
return cmpNumbers((uint64_t)TyL, (uint64_t)TyR);
case Type::VoidTyID:
case Type::FloatTyID:
case Type::DoubleTyID:
case Type::X86_FP80TyID:
case Type::FP128TyID:
case Type::PPC_FP128TyID:
case Type::LabelTyID:
case Type::MetadataTyID:
return 0;
case Type::PointerTyID: {
assert(PTyL && PTyR && "Both types must be pointers here.");
return cmpNumbers(PTyL->getAddressSpace(), PTyR->getAddressSpace());
}
case Type::StructTyID: {
StructType *STyL = cast<StructType>(TyL);
StructType *STyR = cast<StructType>(TyR);
if (STyL->getNumElements() != STyR->getNumElements())
return cmpNumbers(STyL->getNumElements(), STyR->getNumElements());
if (STyL->isPacked() != STyR->isPacked())
return cmpNumbers(STyL->isPacked(), STyR->isPacked());
for (unsigned i = 0, e = STyL->getNumElements(); i != e; ++i) {
if (int Res = cmpType(STyL->getElementType(i),
STyR->getElementType(i)))
return Res;
}
return 0;
}
case Type::FunctionTyID: {
FunctionType *FTyL = cast<FunctionType>(TyL);
FunctionType *FTyR = cast<FunctionType>(TyR);
if (FTyL->getNumParams() != FTyR->getNumParams())
return cmpNumbers(FTyL->getNumParams(), FTyR->getNumParams());
if (FTyL->isVarArg() != FTyR->isVarArg())
return cmpNumbers(FTyL->isVarArg(), FTyR->isVarArg());
if (int Res = cmpType(FTyL->getReturnType(), FTyR->getReturnType()))
return Res;
for (unsigned i = 0, e = FTyL->getNumParams(); i != e; ++i) {
if (int Res = cmpType(FTyL->getParamType(i), FTyR->getParamType(i)))
return Res;
}
return 0;
}
case Type::ArrayTyID: {
ArrayType *ATyL = cast<ArrayType>(TyL);
ArrayType *ATyR = cast<ArrayType>(TyR);
if (ATyL->getNumElements() != ATyR->getNumElements())
return cmpNumbers(ATyL->getNumElements(), ATyR->getNumElements());
return cmpType(ATyL->getElementType(), ATyR->getElementType());
}
}
}
void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
// Check to see if this is a special global used by LLVM, if so, emit it.
if (EmitSpecialLLVMGlobal(gv))
return;
MCSymbol *gvsym = Mang->getSymbol(gv);
assert(gvsym->isUndefined() && "Cannot define a symbol twice!");
SmallString<128> decl;
raw_svector_ostream os(decl);
// check if it is defined in some other translation unit
if (gv->isDeclaration())
os << ".extern ";
// state space: e.g., .global
os << '.' << getStateSpaceName(gv->getType()->getAddressSpace()) << ' ';
// alignment (optional)
unsigned alignment = gv->getAlignment();
if (alignment != 0)
os << ".align " << gv->getAlignment() << ' ';
if (PointerType::classof(gv->getType())) {
PointerType* pointerTy = dyn_cast<PointerType>(gv->getType());
Type* elementTy = pointerTy->getElementType();
if (elementTy->isArrayTy()) {
assert(elementTy->isArrayTy() && "Only pointers to arrays are supported");
ArrayType* arrayTy = dyn_cast<ArrayType>(elementTy);
elementTy = arrayTy->getElementType();
unsigned numElements = arrayTy->getNumElements();
while (elementTy->isArrayTy()) {
arrayTy = dyn_cast<ArrayType>(elementTy);
elementTy = arrayTy->getElementType();
numElements *= arrayTy->getNumElements();
}
// FIXME: isPrimitiveType() == false for i16?
assert(elementTy->isSingleValueType() &&
"Non-primitive types are not handled");
// Find the size of the element in bits
unsigned elementSize = elementTy->getPrimitiveSizeInBits();
os << ".b" << elementSize << ' ' << gvsym->getName()
<< '[' << numElements << ']';
} else {
os << ".b8" << gvsym->getName() << "[]";
}
// handle string constants (assume ConstantArray means string)
if (gv->hasInitializer()) {
const Constant *C = gv->getInitializer();
if (const ConstantArray *CA = dyn_cast<ConstantArray>(C)) {
os << " = {";
for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i) {
if (i > 0)
os << ',';
os << "0x";
os.write_hex(cast<ConstantInt>(CA->getOperand(i))->getZExtValue());
}
os << '}';
}
}
} else {
// Note: this is currently the fall-through case and most likely generates
// incorrect code.
os << getTypeName(gv->getType()) << ' ' << gvsym->getName();
if (isa<ArrayType>(gv->getType()) || isa<PointerType>(gv->getType()))
os << "[]";
}
os << ';';
OutStreamer.EmitRawText(os.str());
OutStreamer.AddBlankLine();
}