static void readFuncList(GlobalVariable *Array, std::vector<Constant*> *Funcs) {
if (!Array->hasInitializer())
return;
Constant *Init = Array->getInitializer();
ArrayType *Ty = dyn_cast<ArrayType>(Init->getType());
if (!Ty) {
errs() << "Initializer: " << *Array->getInitializer() << "\n";
report_fatal_error("ExpandCtors: Initializer is not of array type");
}
if (Ty->getNumElements() == 0)
return;
ConstantArray *InitList = dyn_cast<ConstantArray>(Init);
if (!InitList) {
errs() << "Initializer: " << *Array->getInitializer() << "\n";
report_fatal_error("ExpandCtors: Unexpected initializer ConstantExpr");
}
std::vector<FuncArrayEntry> FuncsToSort;
for (unsigned Index = 0; Index < InitList->getNumOperands(); ++Index) {
ConstantStruct *CS = cast<ConstantStruct>(InitList->getOperand(Index));
FuncArrayEntry Entry;
Entry.priority = cast<ConstantInt>(CS->getOperand(0))->getZExtValue();
Entry.func = CS->getOperand(1);
FuncsToSort.push_back(Entry);
}
std::sort(FuncsToSort.begin(), FuncsToSort.end(), compareEntries);
for (std::vector<FuncArrayEntry>::iterator Iter = FuncsToSort.begin();
Iter != FuncsToSort.end();
++Iter) {
Funcs->push_back(Iter->func);
}
}
static bool SplitUpArrayLoad(LoadInst *Load, const DataLayout *DL) {
ArrayType *ATy = cast<ArrayType>(Load->getType());
Value *NewStruct = UndefValue::get(ATy);
bool NeedsAnotherPass = false;
// Create a separate load instruction for each struct field.
for (unsigned Index = 0; Index < ATy->getNumElements(); ++Index) {
SmallVector<Value *, 2> Indexes;
Indexes.push_back(ConstantInt::get(Load->getContext(), APInt(32, 0)));
Indexes.push_back(ConstantInt::get(Load->getContext(), APInt(32, Index)));
Value *GEP =
CopyDebug(GetElementPtrInst::Create(ATy, Load->getPointerOperand(), Indexes,
Load->getName() + ".index", Load),
Load);
LoadInst *NewLoad = new LoadInst(GEP, Load->getName() + ".field", Load);
NeedsAnotherPass = NeedsAnotherPass || DoAnotherPass(NewLoad);
ProcessArrayLoadOrStoreAttrs(NewLoad, Load, ATy, Index, DL);
// Reconstruct the struct value.
SmallVector<unsigned, 1> EVIndexes;
EVIndexes.push_back(Index);
NewStruct =
CopyDebug(InsertValueInst::Create(NewStruct, NewLoad, EVIndexes,
Load->getName() + ".insert", Load),
Load);
}
Load->replaceAllUsesWith(NewStruct);
Load->eraseFromParent();
return NeedsAnotherPass;
}
static bool SplitUpArrayStore(StoreInst *Store, const DataLayout *DL) {
ArrayType *ATy = cast<ArrayType>(Store->getValueOperand()->getType());
bool NeedsAnotherPass = false;
// Create a separate store instruction for each struct field.
for (unsigned Index = 0; Index < ATy->getNumElements(); ++Index) {
SmallVector<Value *, 2> Indexes;
Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, 0)));
Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, Index)));
Value *GEP =
CopyDebug(GetElementPtrInst::Create(ATy,
Store->getPointerOperand(), Indexes,
Store->getPointerOperand()->getName() + ".index", Store),
Store);
NeedsAnotherPass =
NeedsAnotherPass || DoAnotherPass(GEP->getType()->getContainedType(0));
SmallVector<unsigned, 1> EVIndexes;
EVIndexes.push_back(Index);
Value *Field = ExtractValueInst::Create(Store->getValueOperand(), EVIndexes,
"", Store);
StoreInst *NewStore = new StoreInst(Field, GEP, Store);
ProcessArrayLoadOrStoreAttrs(NewStore, Store, ATy, Index, DL);
}
Store->eraseFromParent();
return NeedsAnotherPass;
}
/// isDereferenceablePointer - Test if this value is always a pointer to
/// allocated and suitably aligned memory for a simple load or store.
static bool isDereferenceablePointer(const Value *V,
SmallPtrSet<const Value *, 32> &Visited) {
// Note that it is not safe to speculate into a malloc'd region because
// malloc may return null.
// It's also not always safe to follow a bitcast, for example:
// bitcast i8* (alloca i8) to i32*
// would result in a 4-byte load from a 1-byte alloca. Some cases could
// be handled using DataLayout to check sizes and alignments though.
// These are obviously ok.
if (isa<AllocaInst>(V)) return true;
// Global variables which can't collapse to null are ok.
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
return !GV->hasExternalWeakLinkage();
// byval arguments are ok.
if (const Argument *A = dyn_cast<Argument>(V))
return A->hasByValAttr();
// For GEPs, determine if the indexing lands within the allocated object.
if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
// Conservatively require that the base pointer be fully dereferenceable.
if (!Visited.insert(GEP->getOperand(0)))
return false;
if (!isDereferenceablePointer(GEP->getOperand(0), Visited))
return false;
// Check the indices.
gep_type_iterator GTI = gep_type_begin(GEP);
for (User::const_op_iterator I = GEP->op_begin()+1,
E = GEP->op_end(); I != E; ++I) {
Value *Index = *I;
Type *Ty = *GTI++;
// Struct indices can't be out of bounds.
if (isa<StructType>(Ty))
continue;
ConstantInt *CI = dyn_cast<ConstantInt>(Index);
if (!CI)
return false;
// Zero is always ok.
if (CI->isZero())
continue;
// Check to see that it's within the bounds of an array.
ArrayType *ATy = dyn_cast<ArrayType>(Ty);
if (!ATy)
return false;
if (CI->getValue().getActiveBits() > 64)
return false;
if (CI->getZExtValue() >= ATy->getNumElements())
return false;
}
// Indices check out; this is dereferenceable.
return true;
}
// If we don't know, assume the worst.
return false;
}
ConstantInt* Variables::getSizeInBits(Type *type) {
ConstantInt *size = NULL;
ArrayType *arrayType = dyn_cast<ArrayType>(type);
switch (arrayType->getElementType()->getTypeID()) {
case Type::FloatTyID: {
int isize = arrayType->getNumElements() * 32;
size = getInt64(isize);
break;
}
case Type::DoubleTyID: {
int isize = arrayType->getNumElements() * 64;
size = getInt64(isize);
break;
}
default:
errs() << "WARNING: Unhandled type @ getSizeInBits\n";
exit(1);
}
return size;
}
uint64_t TargetData::getTypeSizeInBits(Type *Ty) const {
assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
switch (Ty->getTypeID()) {
case Type::LabelTyID:
case Type::PointerTyID:
return getPointerSizeInBits();
case Type::ArrayTyID: {
ArrayType *ATy = cast<ArrayType>(Ty);
return getTypeAllocSizeInBits(ATy->getElementType())*ATy->getNumElements();
}
case Type::StructTyID:
// Get the layout annotation... which is lazily created on demand.
return getStructLayout(cast<StructType>(Ty))->getSizeInBits();
case Type::IntegerTyID:
return cast<IntegerType>(Ty)->getBitWidth();
case Type::VoidTyID:
return 8;
case Type::HalfTyID:
return 16;
case Type::FloatTyID:
return 32;
case Type::DoubleTyID:
case Type::X86_MMXTyID:
return 64;
case Type::PPC_FP128TyID:
case Type::FP128TyID:
return 128;
// In memory objects this is always aligned to a higher boundary, but
// only 80 bits contain information.
case Type::X86_FP80TyID:
return 80;
case Type::VectorTyID:
return cast<VectorType>(Ty)->getBitWidth();
default:
llvm_unreachable("TargetData::getTypeSizeInBits(): Unsupported type");
}
}
static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt)
{
// assumes `jl_isbits(bt)`.
// `ptr` can point to a inline field, do not read the tag from it.
// make sure to return exactly the type specified by
// julia_type_to_llvm as this will be assumed by the callee.
if (bt == jl_bool_type)
return ConstantInt::get(T_int8, (*(const uint8_t*)ptr) ? 1 : 0);
if (jl_is_vecelement_type((jl_value_t*)bt))
bt = (jl_datatype_t*)jl_tparam0(bt);
Type *lt = julia_struct_to_llvm((jl_value_t*)bt, NULL, NULL);
if (type_is_ghost(lt))
return UndefValue::get(NoopType);
if (jl_is_primitivetype(bt)) {
if (lt->isFloatTy()) {
uint32_t data32 = *(const uint32_t*)ptr;
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), APInt(32, data32)));
}
if (lt->isDoubleTy()) {
uint64_t data64 = *(const uint64_t*)ptr;
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), APInt(64, data64)));
}
int nb = jl_datatype_size(bt);
APInt val(8 * nb, 0);
void *bits = const_cast<uint64_t*>(val.getRawData());
assert(sys::IsLittleEndianHost);
memcpy(bits, ptr, nb);
if (lt->isFloatingPointTy()) {
return ConstantFP::get(jl_LLVMContext,
APFloat(lt->getFltSemantics(), val));
}
assert(cast<IntegerType>(lt)->getBitWidth() == 8u * nb);
return ConstantInt::get(lt, val);
}
CompositeType *lct = cast<CompositeType>(lt);
size_t nf = jl_datatype_nfields(bt);
std::vector<Constant*> fields(nf);
for (size_t i = 0; i < nf; i++) {
size_t offs = jl_field_offset(bt, i);
assert(!jl_field_isptr(bt, i));
jl_value_t *ft = jl_field_type(bt, i);
Type *lft = lct->getTypeAtIndex(i);
const uint8_t *ov = (const uint8_t*)ptr + offs;
Constant *val;
if (jl_is_uniontype(ft)) {
// compute the same type layout as julia_struct_to_llvm
size_t fsz = jl_field_size(bt, i);
size_t al = jl_field_align(bt, i);
uint8_t sel = ((const uint8_t*)ptr)[offs + fsz - 1];
jl_value_t *active_ty = jl_nth_union_component(ft, sel);
size_t active_sz = jl_datatype_size(active_ty);
ArrayType *aty = cast<ArrayType>(cast<StructType>(lft)->getTypeAtIndex(0u));
assert(aty->getElementType() == IntegerType::get(jl_LLVMContext, 8 * al) &&
aty->getNumElements() == (fsz - 1) / al);
std::vector<Constant*> ArrayElements(0);
for (unsigned j = 0; j < aty->getNumElements(); j++) {
APInt Elem(8 * al, 0);
void *bits = const_cast<uint64_t*>(Elem.getRawData());
if (active_sz > al) {
memcpy(bits, ov, al);
active_sz -= al;
}
else if (active_sz > 0) {
memcpy(bits, ov, active_sz);
active_sz = 0;
}
ov += al;
ArrayElements.push_back(ConstantInt::get(aty->getElementType(), Elem));
}
std::vector<Constant*> Elements(0);
Elements.push_back(ConstantArray::get(aty, ArrayElements));
unsigned remainder = (fsz - 1) % al;
while (remainder--) {
uint8_t byte;
if (active_sz > 0) {
byte = *ov;
active_sz -= 1;
}
else {
byte = 0;
}
ov += 1;
APInt Elem(8, byte);
Elements.push_back(ConstantInt::get(T_int8, Elem));
}
Elements.push_back(ConstantInt::get(T_int8, sel));
val = ConstantStruct::get(cast<StructType>(lft), Elements);
}
else {
val = julia_const_to_llvm(ov, (jl_datatype_t*)ft);
}
fields[i] = val;
}
if (lct->isVectorTy())
return ConstantVector::get(fields);
if (StructType *st = dyn_cast<StructType>(lct))
return ConstantStruct::get(st, fields);
ArrayType *at = cast<ArrayType>(lct);
return ConstantArray::get(at, fields);
}
static bool tryPromoteAllocaToVector(AllocaInst *Alloca, AMDGPUAS AS) {
ArrayType *AllocaTy = dyn_cast<ArrayType>(Alloca->getAllocatedType());
DEBUG(dbgs() << "Alloca candidate for vectorization\n");
// FIXME: There is no reason why we can't support larger arrays, we
// are just being conservative for now.
// FIXME: We also reject alloca's of the form [ 2 x [ 2 x i32 ]] or equivalent. Potentially these
// could also be promoted but we don't currently handle this case
if (!AllocaTy ||
AllocaTy->getNumElements() > 4 ||
AllocaTy->getNumElements() < 2 ||
!VectorType::isValidElementType(AllocaTy->getElementType())) {
DEBUG(dbgs() << " Cannot convert type to vector\n");
return false;
}
std::map<GetElementPtrInst*, Value*> GEPVectorIdx;
std::vector<Value*> WorkList;
for (User *AllocaUser : Alloca->users()) {
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
if (!GEP) {
if (!canVectorizeInst(cast<Instruction>(AllocaUser), Alloca))
return false;
WorkList.push_back(AllocaUser);
continue;
}
Value *Index = GEPToVectorIndex(GEP);
// If we can't compute a vector index from this GEP, then we can't
// promote this alloca to vector.
if (!Index) {
DEBUG(dbgs() << " Cannot compute vector index for GEP " << *GEP << '\n');
return false;
}
GEPVectorIdx[GEP] = Index;
for (User *GEPUser : AllocaUser->users()) {
if (!canVectorizeInst(cast<Instruction>(GEPUser), AllocaUser))
return false;
WorkList.push_back(GEPUser);
}
}
VectorType *VectorTy = arrayTypeToVecType(AllocaTy);
DEBUG(dbgs() << " Converting alloca to vector "
<< *AllocaTy << " -> " << *VectorTy << '\n');
for (Value *V : WorkList) {
Instruction *Inst = cast<Instruction>(V);
IRBuilder<> Builder(Inst);
switch (Inst->getOpcode()) {
case Instruction::Load: {
Type *VecPtrTy = VectorTy->getPointerTo(AS.PRIVATE_ADDRESS);
Value *Ptr = cast<LoadInst>(Inst)->getPointerOperand();
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
Value *VecValue = Builder.CreateLoad(BitCast);
Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
Inst->replaceAllUsesWith(ExtractElement);
Inst->eraseFromParent();
break;
}
case Instruction::Store: {
Type *VecPtrTy = VectorTy->getPointerTo(AS.PRIVATE_ADDRESS);
StoreInst *SI = cast<StoreInst>(Inst);
Value *Ptr = SI->getPointerOperand();
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
Value *VecValue = Builder.CreateLoad(BitCast);
Value *NewVecValue = Builder.CreateInsertElement(VecValue,
SI->getValueOperand(),
Index);
Builder.CreateStore(NewVecValue, BitCast);
Inst->eraseFromParent();
break;
}
case Instruction::BitCast:
case Instruction::AddrSpaceCast:
break;
default:
llvm_unreachable("Inconsistency in instructions promotable to vector");
}
}
return true;
}
static bool isZeroLengthArray(Type *Ty) {
ArrayType *AT = dyn_cast<ArrayType>(Ty);
return AT && (AT->getNumElements() == 0);
}
/// isDereferenceablePointer - Test if this value is always a pointer to
/// allocated and suitably aligned memory for a simple load or store.
static bool isDereferenceablePointer(const Value *V, const DataLayout *DL,
SmallPtrSetImpl<const Value *> &Visited) {
// Note that it is not safe to speculate into a malloc'd region because
// malloc may return null.
// These are obviously ok.
if (isa<AllocaInst>(V)) return true;
// It's not always safe to follow a bitcast, for example:
// bitcast i8* (alloca i8) to i32*
// would result in a 4-byte load from a 1-byte alloca. However,
// if we're casting from a pointer from a type of larger size
// to a type of smaller size (or the same size), and the alignment
// is at least as large as for the resulting pointer type, then
// we can look through the bitcast.
if (DL)
if (const BitCastInst* BC = dyn_cast<BitCastInst>(V)) {
Type *STy = BC->getSrcTy()->getPointerElementType(),
*DTy = BC->getDestTy()->getPointerElementType();
if (STy->isSized() && DTy->isSized() &&
(DL->getTypeStoreSize(STy) >=
DL->getTypeStoreSize(DTy)) &&
(DL->getABITypeAlignment(STy) >=
DL->getABITypeAlignment(DTy)))
return isDereferenceablePointer(BC->getOperand(0), DL, Visited);
}
// Global variables which can't collapse to null are ok.
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
return !GV->hasExternalWeakLinkage();
// byval arguments are okay. Arguments specifically marked as
// dereferenceable are okay too.
if (const Argument *A = dyn_cast<Argument>(V)) {
if (A->hasByValAttr())
return true;
else if (uint64_t Bytes = A->getDereferenceableBytes()) {
Type *Ty = V->getType()->getPointerElementType();
if (Ty->isSized() && DL && DL->getTypeStoreSize(Ty) <= Bytes)
return true;
}
return false;
}
// Return values from call sites specifically marked as dereferenceable are
// also okay.
if (ImmutableCallSite CS = V) {
if (uint64_t Bytes = CS.getDereferenceableBytes(0)) {
Type *Ty = V->getType()->getPointerElementType();
if (Ty->isSized() && DL && DL->getTypeStoreSize(Ty) <= Bytes)
return true;
}
}
// For GEPs, determine if the indexing lands within the allocated object.
if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
// Conservatively require that the base pointer be fully dereferenceable.
if (!Visited.insert(GEP->getOperand(0)))
return false;
if (!isDereferenceablePointer(GEP->getOperand(0), DL, Visited))
return false;
// Check the indices.
gep_type_iterator GTI = gep_type_begin(GEP);
for (User::const_op_iterator I = GEP->op_begin()+1,
E = GEP->op_end(); I != E; ++I) {
Value *Index = *I;
Type *Ty = *GTI++;
// Struct indices can't be out of bounds.
if (isa<StructType>(Ty))
continue;
ConstantInt *CI = dyn_cast<ConstantInt>(Index);
if (!CI)
return false;
// Zero is always ok.
if (CI->isZero())
continue;
// Check to see that it's within the bounds of an array.
ArrayType *ATy = dyn_cast<ArrayType>(Ty);
if (!ATy)
return false;
if (CI->getValue().getActiveBits() > 64)
return false;
if (CI->getZExtValue() >= ATy->getNumElements())
return false;
}
// Indices check out; this is dereferenceable.
return true;
}
if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V))
return isDereferenceablePointer(ASC->getOperand(0), DL, Visited);
// If we don't know, assume the worst.
return false;
}
void visitAll( const ArrayType& a ) {
const ValueType *v = static_cast< const ValueType* >( a.getDataPointer() );
unsigned int size = a.getNumElements();
for( unsigned int i=0; i<size; i++ )
valueVisitor->apply( v[i] );
}
/// If there were any appending global variables, link them together now.
Expected<Constant *>
IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,
const GlobalVariable *SrcGV) {
Type *EltTy = cast<ArrayType>(TypeMap.get(SrcGV->getValueType()))
->getElementType();
// FIXME: This upgrade is done during linking to support the C API. Once the
// old form is deprecated, we should move this upgrade to
// llvm::UpgradeGlobalVariable() and simplify the logic here and in
// Mapper::mapAppendingVariable() in ValueMapper.cpp.
StringRef Name = SrcGV->getName();
bool IsNewStructor = false;
bool IsOldStructor = false;
if (Name == "llvm.global_ctors" || Name == "llvm.global_dtors") {
if (cast<StructType>(EltTy)->getNumElements() == 3)
IsNewStructor = true;
else
IsOldStructor = true;
}
PointerType *VoidPtrTy = Type::getInt8Ty(SrcGV->getContext())->getPointerTo();
if (IsOldStructor) {
auto &ST = *cast<StructType>(EltTy);
Type *Tys[3] = {ST.getElementType(0), ST.getElementType(1), VoidPtrTy};
EltTy = StructType::get(SrcGV->getContext(), Tys, false);
}
uint64_t DstNumElements = 0;
if (DstGV) {
ArrayType *DstTy = cast<ArrayType>(DstGV->getValueType());
DstNumElements = DstTy->getNumElements();
if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage())
return stringErr(
"Linking globals named '" + SrcGV->getName() +
"': can only link appending global with another appending "
"global!");
// Check to see that they two arrays agree on type.
if (EltTy != DstTy->getElementType())
return stringErr("Appending variables with different element types!");
if (DstGV->isConstant() != SrcGV->isConstant())
return stringErr("Appending variables linked with different const'ness!");
if (DstGV->getAlignment() != SrcGV->getAlignment())
return stringErr(
"Appending variables with different alignment need to be linked!");
if (DstGV->getVisibility() != SrcGV->getVisibility())
return stringErr(
"Appending variables with different visibility need to be linked!");
if (DstGV->hasGlobalUnnamedAddr() != SrcGV->hasGlobalUnnamedAddr())
return stringErr(
"Appending variables with different unnamed_addr need to be linked!");
if (DstGV->getSection() != SrcGV->getSection())
return stringErr(
"Appending variables with different section name need to be linked!");
}
SmallVector<Constant *, 16> SrcElements;
getArrayElements(SrcGV->getInitializer(), SrcElements);
if (IsNewStructor) {
auto It = remove_if(SrcElements, [this](Constant *E) {
auto *Key =
dyn_cast<GlobalValue>(E->getAggregateElement(2)->stripPointerCasts());
if (!Key)
return false;
GlobalValue *DGV = getLinkedToGlobal(Key);
return !shouldLink(DGV, *Key);
});
SrcElements.erase(It, SrcElements.end());
}
uint64_t NewSize = DstNumElements + SrcElements.size();
ArrayType *NewType = ArrayType::get(EltTy, NewSize);
// Create the new global variable.
GlobalVariable *NG = new GlobalVariable(
DstM, NewType, SrcGV->isConstant(), SrcGV->getLinkage(),
/*init*/ nullptr, /*name*/ "", DstGV, SrcGV->getThreadLocalMode(),
SrcGV->getType()->getAddressSpace());
NG->copyAttributesFrom(SrcGV);
forceRenaming(NG, SrcGV->getName());
Constant *Ret = ConstantExpr::getBitCast(NG, TypeMap.get(SrcGV->getType()));
Mapper.scheduleMapAppendingVariable(*NG,
DstGV ? DstGV->getInitializer() : nullptr,
IsOldStructor, SrcElements);
// Replace any uses of the two global variables with uses of the new
// global.
if (DstGV) {
DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
DstGV->eraseFromParent();
}
return Ret;
}
/// linkAppendingVarProto - If there were any appending global variables, link
/// them together now. Return true on error.
bool ModuleLinker::linkAppendingVarProto(GlobalVariable *DstGV,
GlobalVariable *SrcGV) {
if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage())
return emitError("Linking globals named '" + SrcGV->getName() +
"': can only link appending global with another appending global!");
ArrayType *DstTy = cast<ArrayType>(DstGV->getType()->getElementType());
ArrayType *SrcTy =
cast<ArrayType>(TypeMap.get(SrcGV->getType()->getElementType()));
Type *EltTy = DstTy->getElementType();
// Check to see that they two arrays agree on type.
if (EltTy != SrcTy->getElementType())
return emitError("Appending variables with different element types!");
if (DstGV->isConstant() != SrcGV->isConstant())
return emitError("Appending variables linked with different const'ness!");
if (DstGV->getAlignment() != SrcGV->getAlignment())
return emitError(
"Appending variables with different alignment need to be linked!");
if (DstGV->getVisibility() != SrcGV->getVisibility())
return emitError(
"Appending variables with different visibility need to be linked!");
if (DstGV->getSection() != SrcGV->getSection())
return emitError(
"Appending variables with different section name need to be linked!");
uint64_t NewSize = DstTy->getNumElements() + SrcTy->getNumElements();
ArrayType *NewType = ArrayType::get(EltTy, NewSize);
// Create the new global variable.
GlobalVariable *NG =
new GlobalVariable(*DstGV->getParent(), NewType, SrcGV->isConstant(),
DstGV->getLinkage(), /*init*/0, /*name*/"", DstGV,
DstGV->isThreadLocal(),
DstGV->getType()->getAddressSpace());
// Propagate alignment, visibility and section info.
CopyGVAttributes(NG, DstGV);
AppendingVarInfo AVI;
AVI.NewGV = NG;
AVI.DstInit = DstGV->getInitializer();
AVI.SrcInit = SrcGV->getInitializer();
AppendingVars.push_back(AVI);
// Replace any uses of the two global variables with uses of the new
// global.
ValueMap[SrcGV] = ConstantExpr::getBitCast(NG, TypeMap.get(SrcGV->getType()));
DstGV->replaceAllUsesWith(ConstantExpr::getBitCast(NG, DstGV->getType()));
DstGV->eraseFromParent();
// Track the source variable so we don't try to link it.
DoNotLinkFromSource.insert(SrcGV);
return false;
}
static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
ArrayType *AllocaTy = dyn_cast<ArrayType>(Alloca->getAllocatedType());
DEBUG(dbgs() << "Alloca candidate for vectorization\n");
// FIXME: There is no reason why we can't support larger arrays, we
// are just being conservative for now.
if (!AllocaTy ||
AllocaTy->getElementType()->isVectorTy() ||
AllocaTy->getNumElements() > 4) {
DEBUG(dbgs() << " Cannot convert type to vector\n");
return false;
}
std::map<GetElementPtrInst*, Value*> GEPVectorIdx;
std::vector<Value*> WorkList;
for (User *AllocaUser : Alloca->users()) {
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
if (!GEP) {
if (!canVectorizeInst(cast<Instruction>(AllocaUser), Alloca))
return false;
WorkList.push_back(AllocaUser);
continue;
}
Value *Index = GEPToVectorIndex(GEP);
// If we can't compute a vector index from this GEP, then we can't
// promote this alloca to vector.
if (!Index) {
DEBUG(dbgs() << " Cannot compute vector index for GEP " << *GEP << '\n');
return false;
}
GEPVectorIdx[GEP] = Index;
for (User *GEPUser : AllocaUser->users()) {
if (!canVectorizeInst(cast<Instruction>(GEPUser), AllocaUser))
return false;
WorkList.push_back(GEPUser);
}
}
VectorType *VectorTy = arrayTypeToVecType(AllocaTy);
DEBUG(dbgs() << " Converting alloca to vector "
<< *AllocaTy << " -> " << *VectorTy << '\n');
for (Value *V : WorkList) {
Instruction *Inst = cast<Instruction>(V);
IRBuilder<> Builder(Inst);
switch (Inst->getOpcode()) {
case Instruction::Load: {
Value *Ptr = Inst->getOperand(0);
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
Value *VecValue = Builder.CreateLoad(BitCast);
Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
Inst->replaceAllUsesWith(ExtractElement);
Inst->eraseFromParent();
break;
}
case Instruction::Store: {
Value *Ptr = Inst->getOperand(1);
Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
Value *VecValue = Builder.CreateLoad(BitCast);
Value *NewVecValue = Builder.CreateInsertElement(VecValue,
Inst->getOperand(0),
Index);
Builder.CreateStore(NewVecValue, BitCast);
Inst->eraseFromParent();
break;
}
case Instruction::BitCast:
case Instruction::AddrSpaceCast:
break;
default:
Inst->dump();
llvm_unreachable("Inconsistency in instructions promotable to vector");
}
}
return true;
}
bool LowerEmExceptions::runOnModule(Module &M) {
TheModule = &M;
// Add functions
Type *i32 = Type::getInt32Ty(M.getContext());
Type *i8 = Type::getInt8Ty(M.getContext());
Type *i1 = Type::getInt1Ty(M.getContext());
Type *i8P = i8->getPointerTo();
Type *Void = Type::getVoidTy(M.getContext());
if (!(GetHigh = TheModule->getFunction("getHigh32"))) {
FunctionType *GetHighFunc = FunctionType::get(i32, false);
GetHigh = Function::Create(GetHighFunc, GlobalValue::ExternalLinkage,
"getHigh32", TheModule);
}
if (!(PreInvoke = TheModule->getFunction("emscripten_preinvoke"))) {
FunctionType *VoidFunc = FunctionType::get(Void, false);
PreInvoke = Function::Create(VoidFunc, GlobalValue::ExternalLinkage, "emscripten_preinvoke", TheModule);
}
if (!(PostInvoke = TheModule->getFunction("emscripten_postinvoke"))) {
FunctionType *IntFunc = FunctionType::get(i32, false);
PostInvoke = Function::Create(IntFunc, GlobalValue::ExternalLinkage, "emscripten_postinvoke", TheModule);
}
FunctionType *LandingPadFunc = FunctionType::get(i8P, true);
LandingPad = Function::Create(LandingPadFunc, GlobalValue::ExternalLinkage, "emscripten_landingpad", TheModule);
FunctionType *ResumeFunc = FunctionType::get(Void, true);
Resume = Function::Create(ResumeFunc, GlobalValue::ExternalLinkage, "emscripten_resume", TheModule);
// Process
bool HasWhitelist = Whitelist.size() > 0;
std::string WhitelistChecker;
if (HasWhitelist) WhitelistChecker = "," + Whitelist + ",";
bool Changed = false;
for (Module::iterator Iter = M.begin(), E = M.end(); Iter != E; ) {
Function *F = Iter++;
std::vector<Instruction*> ToErase;
std::set<LandingPadInst*> LandingPads;
bool AllowExceptionsInFunc = !HasWhitelist || int(WhitelistChecker.find(F->getName())) > 0;
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
// check terminator for invokes
if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) {
LandingPads.insert(II->getLandingPadInst());
bool NeedInvoke = AllowExceptionsInFunc && canThrow(II->getCalledValue());
if (NeedInvoke) {
// Insert a normal call instruction folded in between pre- and post-invoke
CallInst::Create(PreInvoke, "", II);
SmallVector<Value*,16> CallArgs(II->op_begin(), II->op_end() - 3);
CallInst *NewCall = CallInst::Create(II->getCalledValue(),
CallArgs, "", II);
NewCall->takeName(II);
NewCall->setCallingConv(II->getCallingConv());
NewCall->setAttributes(II->getAttributes());
NewCall->setDebugLoc(II->getDebugLoc());
II->replaceAllUsesWith(NewCall);
ToErase.push_back(II);
CallInst *Post = CallInst::Create(PostInvoke, "", II);
Instruction *Post1 = new TruncInst(Post, i1, "", II);
// Insert a branch based on the postInvoke
BranchInst::Create(II->getUnwindDest(), II->getNormalDest(), Post1, II);
} else {
// This can't throw, and we don't need this invoke, just replace it with a call+branch
SmallVector<Value*,16> CallArgs(II->op_begin(), II->op_end() - 3);
CallInst *NewCall = CallInst::Create(II->getCalledValue(),
CallArgs, "", II);
NewCall->takeName(II);
NewCall->setCallingConv(II->getCallingConv());
NewCall->setAttributes(II->getAttributes());
NewCall->setDebugLoc(II->getDebugLoc());
II->replaceAllUsesWith(NewCall);
ToErase.push_back(II);
BranchInst::Create(II->getNormalDest(), II);
// Remove any PHI node entries from the exception destination.
II->getUnwindDest()->removePredecessor(BB);
}
Changed = true;
}
// scan the body of the basic block for resumes
for (BasicBlock::iterator Iter = BB->begin(), E = BB->end();
Iter != E; ) {
Instruction *I = Iter++;
if (ResumeInst *R = dyn_cast<ResumeInst>(I)) {
// split the input into legal values
Value *Input = R->getValue();
ExtractValueInst *Low = ExtractValueInst::Create(Input, 0, "", R);
ExtractValueInst *High = ExtractValueInst::Create(Input, 1, "", R);
// create a resume call
SmallVector<Value*,2> CallArgs;
CallArgs.push_back(Low);
CallArgs.push_back(High);
CallInst::Create(Resume, CallArgs, "", R);
new UnreachableInst(TheModule->getContext(), R); // add a terminator to the block
ToErase.push_back(R);
}
}
}
// Look for orphan landingpads, can occur in blocks with no predecesors
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
Instruction *I = BB->getFirstNonPHI();
if (LandingPadInst *LP = dyn_cast<LandingPadInst>(I)) {
LandingPads.insert(LP);
}
}
// Handle all the landingpad for this function together, as multiple invokes may share a single lp
for (std::set<LandingPadInst*>::iterator I = LandingPads.begin(); I != LandingPads.end(); I++) {
// Replace the landingpad with a landingpad call to get the low part, and a getHigh for the high
LandingPadInst *LP = *I;
unsigned Num = LP->getNumClauses();
SmallVector<Value*,16> NewLPArgs;
NewLPArgs.push_back(LP->getPersonalityFn());
for (unsigned i = 0; i < Num; i++) {
Value *Arg = LP->getClause(i);
// As a temporary workaround for the lack of aggregate varargs support
// in the varargs lowering code, break out filter operands into their
// component elements.
if (LP->isFilter(i)) {
ArrayType *ATy = cast<ArrayType>(Arg->getType());
for (unsigned elem = 0, elemEnd = ATy->getNumElements(); elem != elemEnd; ++elem) {
Instruction *EE = ExtractValueInst::Create(Arg, makeArrayRef(elem), "", LP);
NewLPArgs.push_back(EE);
}
} else {
NewLPArgs.push_back(Arg);
}
}
NewLPArgs.push_back(LP->isCleanup() ? ConstantInt::getTrue(i1) : ConstantInt::getFalse(i1));
CallInst *NewLP = CallInst::Create(LandingPad, NewLPArgs, "", LP);
Instruction *High = CallInst::Create(GetHigh, "", LP);
// New recreate an aggregate for them, which will be all simplified later (simplification cannot handle landingpad, hence all this)
InsertValueInst *IVA = InsertValueInst::Create(UndefValue::get(LP->getType()), NewLP, 0, "", LP);
InsertValueInst *IVB = InsertValueInst::Create(IVA, High, 1, "", LP);
LP->replaceAllUsesWith(IVB);
ToErase.push_back(LP);
}
// erase everything we no longer need in this function
for (unsigned i = 0; i < ToErase.size(); i++) ToErase[i]->eraseFromParent();
}
return Changed;
}
/// cmpType - compares two types,
/// defines total ordering among the types set.
/// See method declaration comments for more details.
int FunctionComparator::cmpType(Type *TyL, Type *TyR) const {
PointerType *PTyL = dyn_cast<PointerType>(TyL);
PointerType *PTyR = dyn_cast<PointerType>(TyR);
if (DL) {
if (PTyL && PTyL->getAddressSpace() == 0) TyL = DL->getIntPtrType(TyL);
if (PTyR && PTyR->getAddressSpace() == 0) TyR = DL->getIntPtrType(TyR);
}
if (TyL == TyR)
return 0;
if (int Res = cmpNumbers(TyL->getTypeID(), TyR->getTypeID()))
return Res;
switch (TyL->getTypeID()) {
default:
llvm_unreachable("Unknown type!");
// Fall through in Release mode.
case Type::IntegerTyID:
case Type::VectorTyID:
// TyL == TyR would have returned true earlier.
return cmpNumbers((uint64_t)TyL, (uint64_t)TyR);
case Type::VoidTyID:
case Type::FloatTyID:
case Type::DoubleTyID:
case Type::X86_FP80TyID:
case Type::FP128TyID:
case Type::PPC_FP128TyID:
case Type::LabelTyID:
case Type::MetadataTyID:
return 0;
case Type::PointerTyID: {
assert(PTyL && PTyR && "Both types must be pointers here.");
return cmpNumbers(PTyL->getAddressSpace(), PTyR->getAddressSpace());
}
case Type::StructTyID: {
StructType *STyL = cast<StructType>(TyL);
StructType *STyR = cast<StructType>(TyR);
if (STyL->getNumElements() != STyR->getNumElements())
return cmpNumbers(STyL->getNumElements(), STyR->getNumElements());
if (STyL->isPacked() != STyR->isPacked())
return cmpNumbers(STyL->isPacked(), STyR->isPacked());
for (unsigned i = 0, e = STyL->getNumElements(); i != e; ++i) {
if (int Res = cmpType(STyL->getElementType(i),
STyR->getElementType(i)))
return Res;
}
return 0;
}
case Type::FunctionTyID: {
FunctionType *FTyL = cast<FunctionType>(TyL);
FunctionType *FTyR = cast<FunctionType>(TyR);
if (FTyL->getNumParams() != FTyR->getNumParams())
return cmpNumbers(FTyL->getNumParams(), FTyR->getNumParams());
if (FTyL->isVarArg() != FTyR->isVarArg())
return cmpNumbers(FTyL->isVarArg(), FTyR->isVarArg());
if (int Res = cmpType(FTyL->getReturnType(), FTyR->getReturnType()))
return Res;
for (unsigned i = 0, e = FTyL->getNumParams(); i != e; ++i) {
if (int Res = cmpType(FTyL->getParamType(i), FTyR->getParamType(i)))
return Res;
}
return 0;
}
case Type::ArrayTyID: {
ArrayType *ATyL = cast<ArrayType>(TyL);
ArrayType *ATyR = cast<ArrayType>(TyR);
if (ATyL->getNumElements() != ATyR->getNumElements())
return cmpNumbers(ATyL->getNumElements(), ATyR->getNumElements());
return cmpType(ATyL->getElementType(), ATyR->getElementType());
}
}
}
void PTXAsmPrinter::EmitVariableDeclaration(const GlobalVariable *gv) {
// Check to see if this is a special global used by LLVM, if so, emit it.
if (EmitSpecialLLVMGlobal(gv))
return;
MCSymbol *gvsym = Mang->getSymbol(gv);
assert(gvsym->isUndefined() && "Cannot define a symbol twice!");
SmallString<128> decl;
raw_svector_ostream os(decl);
// check if it is defined in some other translation unit
if (gv->isDeclaration())
os << ".extern ";
// state space: e.g., .global
os << '.' << getStateSpaceName(gv->getType()->getAddressSpace()) << ' ';
// alignment (optional)
unsigned alignment = gv->getAlignment();
if (alignment != 0)
os << ".align " << gv->getAlignment() << ' ';
if (PointerType::classof(gv->getType())) {
PointerType* pointerTy = dyn_cast<PointerType>(gv->getType());
Type* elementTy = pointerTy->getElementType();
if (elementTy->isArrayTy()) {
assert(elementTy->isArrayTy() && "Only pointers to arrays are supported");
ArrayType* arrayTy = dyn_cast<ArrayType>(elementTy);
elementTy = arrayTy->getElementType();
unsigned numElements = arrayTy->getNumElements();
while (elementTy->isArrayTy()) {
arrayTy = dyn_cast<ArrayType>(elementTy);
elementTy = arrayTy->getElementType();
numElements *= arrayTy->getNumElements();
}
// FIXME: isPrimitiveType() == false for i16?
assert(elementTy->isSingleValueType() &&
"Non-primitive types are not handled");
// Find the size of the element in bits
unsigned elementSize = elementTy->getPrimitiveSizeInBits();
os << ".b" << elementSize << ' ' << gvsym->getName()
<< '[' << numElements << ']';
} else {
os << ".b8" << gvsym->getName() << "[]";
}
// handle string constants (assume ConstantArray means string)
if (gv->hasInitializer()) {
const Constant *C = gv->getInitializer();
if (const ConstantArray *CA = dyn_cast<ConstantArray>(C)) {
os << " = {";
for (unsigned i = 0, e = C->getNumOperands(); i != e; ++i) {
if (i > 0)
os << ',';
os << "0x";
os.write_hex(cast<ConstantInt>(CA->getOperand(i))->getZExtValue());
}
os << '}';
}
}
} else {
// Note: this is currently the fall-through case and most likely generates
// incorrect code.
os << getTypeName(gv->getType()) << ' ' << gvsym->getName();
if (isa<ArrayType>(gv->getType()) || isa<PointerType>(gv->getType()))
os << "[]";
}
os << ';';
OutStreamer.EmitRawText(os.str());
OutStreamer.AddBlankLine();
}
