static KernelArg::Metadata getRuntimeMDForKernelArg(const DataLayout &DL,
Type *T, KernelArg::Kind Kind, StringRef BaseTypeName = "",
StringRef TypeName = "", StringRef ArgName = "", StringRef TypeQual = "",
StringRef AccQual = "") {
KernelArg::Metadata Arg;
// Set ArgSize and ArgAlign.
Arg.Size = DL.getTypeAllocSize(T);
Arg.Align = DL.getABITypeAlignment(T);
if (auto PT = dyn_cast<PointerType>(T)) {
auto ET = PT->getElementType();
if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && ET->isSized())
Arg.PointeeAlign = DL.getABITypeAlignment(ET);
}
// Set ArgTypeName.
Arg.TypeName = TypeName;
// Set ArgName.
Arg.Name = ArgName;
// Set ArgIsVolatile, ArgIsRestrict, ArgIsConst and ArgIsPipe.
SmallVector<StringRef, 1> SplitQ;
TypeQual.split(SplitQ, " ", -1, false /* Drop empty entry */);
for (StringRef KeyName : SplitQ) {
auto *P = StringSwitch<uint8_t *>(KeyName)
.Case("volatile", &Arg.IsVolatile)
.Case("restrict", &Arg.IsRestrict)
.Case("const", &Arg.IsConst)
.Case("pipe", &Arg.IsPipe)
.Default(nullptr);
if (P)
*P = 1;
}
// Set ArgKind.
Arg.Kind = Kind;
// Set ArgValueType.
Arg.ValueType = getRuntimeMDValueType(T, BaseTypeName);
// Set ArgAccQual.
if (!AccQual.empty()) {
Arg.AccQual = StringSwitch<KernelArg::AccessQualifer>(AccQual)
.Case("read_only", KernelArg::ReadOnly)
.Case("write_only", KernelArg::WriteOnly)
.Case("read_write", KernelArg::ReadWrite)
.Default(KernelArg::AccNone);
}
// Set ArgAddrQual.
if (auto *PT = dyn_cast<PointerType>(T)) {
Arg.AddrQual = getRuntimeAddrSpace(static_cast<AMDGPUAS::AddressSpaces>(
PT->getAddressSpace()));
}
return Arg;
}
static unsigned findCommonAlignment(const DataLayout &DL, const StoreInst *SI,
const LoadInst *LI) {
unsigned StoreAlign = SI->getAlignment();
if (!StoreAlign)
StoreAlign = DL.getABITypeAlignment(SI->getOperand(0)->getType());
unsigned LoadAlign = LI->getAlignment();
if (!LoadAlign)
LoadAlign = DL.getABITypeAlignment(LI->getType());
return std::min(StoreAlign, LoadAlign);
}
unsigned Value::getPointerAlignment(const DataLayout &DL) const {
assert(getType()->isPointerTy() && "must be pointer");
unsigned Align = 0;
if (auto *GO = dyn_cast<GlobalObject>(this)) {
// Don't make any assumptions about function pointer alignment. Some
// targets use the LSBs to store additional information.
if (isa<Function>(GO))
return 0;
Align = GO->getAlignment();
if (Align == 0) {
if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
Type *ObjectType = GVar->getValueType();
if (ObjectType->isSized()) {
// If the object is defined in the current Module, we'll be giving
// it the preferred alignment. Otherwise, we have to assume that it
// may only have the minimum ABI alignment.
if (GVar->isStrongDefinitionForLinker())
Align = DL.getPreferredAlignment(GVar);
else
Align = DL.getABITypeAlignment(ObjectType);
}
}
}
} else if (const Argument *A = dyn_cast<Argument>(this)) {
Align = A->getParamAlignment();
if (!Align && A->hasStructRetAttr()) {
// An sret parameter has at least the ABI alignment of the return type.
Type *EltTy = cast<PointerType>(A->getType())->getElementType();
if (EltTy->isSized())
Align = DL.getABITypeAlignment(EltTy);
}
} else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) {
Align = AI->getAlignment();
if (Align == 0) {
Type *AllocatedType = AI->getAllocatedType();
if (AllocatedType->isSized())
Align = DL.getPrefTypeAlignment(AllocatedType);
}
} else if (auto CS = ImmutableCallSite(this))
Align = CS.getAttributes().getRetAlignment();
else if (const LoadInst *LI = dyn_cast<LoadInst>(this))
if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) {
ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0));
Align = CI->getLimitedValue();
}
return Align;
}
bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align,
const DataLayout &DL,
const Instruction *CtxI,
const DominatorTree *DT,
const TargetLibraryInfo *TLI) {
// When dereferenceability information is provided by a dereferenceable
// attribute, we know exactly how many bytes are dereferenceable. If we can
// determine the exact offset to the attributed variable, we can use that
// information here.
Type *VTy = V->getType();
Type *Ty = VTy->getPointerElementType();
// Require ABI alignment for loads without alignment specification
if (Align == 0)
Align = DL.getABITypeAlignment(Ty);
if (Ty->isSized()) {
APInt Offset(DL.getTypeStoreSizeInBits(VTy), 0);
const Value *BV = V->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
if (Offset.isNonNegative())
if (isDereferenceableFromAttribute(BV, Offset, Ty, DL, CtxI, DT, TLI) &&
isAligned(BV, Offset, Align, DL))
return true;
}
SmallPtrSet<const Value *, 32> Visited;
return ::isDereferenceableAndAlignedPointer(V, Align, DL, CtxI, DT, TLI,
Visited);
}
StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
assert(!ST->isOpaque() && "Cannot get layout of opaque structs");
StructAlignment = 0;
StructSize = 0;
NumElements = ST->getNumElements();
// Loop over each of the elements, placing them in memory.
for (unsigned i = 0, e = NumElements; i != e; ++i) {
Type *Ty = ST->getElementType(i);
unsigned TyAlign = ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty);
// Add padding if necessary to align the data element properly.
if ((StructSize & (TyAlign-1)) != 0)
StructSize = RoundUpToAlignment(StructSize, TyAlign);
// Keep track of maximum alignment constraint.
StructAlignment = std::max(TyAlign, StructAlignment);
MemberOffsets[i] = StructSize;
StructSize += DL.getTypeAllocSize(Ty); // Consume space for this data item
}
// Empty structures have alignment of 1 byte.
if (StructAlignment == 0) StructAlignment = 1;
// Add padding to the end of the struct so that it could be put in an array
// and all array elements would be aligned correctly.
if ((StructSize & (StructAlignment-1)) != 0)
StructSize = RoundUpToAlignment(StructSize, StructAlignment);
}
static bool isAligned(const Value *Base, APInt Offset, unsigned Align,
const DataLayout &DL) {
APInt BaseAlign(Offset.getBitWidth(), Base->getPointerAlignment(DL));
if (!BaseAlign) {
Type *Ty = Base->getType()->getPointerElementType();
if (!Ty->isSized())
return false;
BaseAlign = DL.getABITypeAlignment(Ty);
}
APInt Alignment(Offset.getBitWidth(), Align);
assert(Alignment.isPowerOf2() && "must be a power of 2!");
return BaseAlign.uge(Alignment) && !(Offset & (Alignment-1));
}
/// getPointeeAlignment - Compute the minimum alignment of the value pointed
/// to by the given pointer.
static unsigned getPointeeAlignment(Value *V, const DataLayout &TD) {
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
if (CE->getOpcode() == Instruction::BitCast ||
(CE->getOpcode() == Instruction::GetElementPtr &&
cast<GEPOperator>(CE)->hasAllZeroIndices()))
return getPointeeAlignment(CE->getOperand(0), TD);
if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
if (!GV->isDeclaration())
return TD.getPreferredAlignment(GV);
if (PointerType *PT = dyn_cast<PointerType>(V->getType()))
return TD.getABITypeAlignment(PT->getElementType());
return 0;
}
// Try to fill in Layout from Ty, returning true on success. Alignment is
// the alignment of the vector, or 0 if the ABI default should be used.
bool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment,
VectorLayout &Layout, const DataLayout &DL) {
// Make sure we're dealing with a vector.
Layout.VecTy = dyn_cast<VectorType>(Ty);
if (!Layout.VecTy)
return false;
// Check that we're dealing with full-byte elements.
Layout.ElemTy = Layout.VecTy->getElementType();
if (DL.getTypeSizeInBits(Layout.ElemTy) !=
DL.getTypeStoreSizeInBits(Layout.ElemTy))
return false;
if (Alignment)
Layout.VecAlign = Alignment;
else
Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy);
Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
return true;
}
bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align,
const DataLayout &DL,
const Instruction *CtxI,
const DominatorTree *DT) {
// When dereferenceability information is provided by a dereferenceable
// attribute, we know exactly how many bytes are dereferenceable. If we can
// determine the exact offset to the attributed variable, we can use that
// information here.
Type *VTy = V->getType();
Type *Ty = VTy->getPointerElementType();
// Require ABI alignment for loads without alignment specification
if (Align == 0)
Align = DL.getABITypeAlignment(Ty);
if (!Ty->isSized())
return false;
SmallPtrSet<const Value *, 32> Visited;
return ::isDereferenceableAndAlignedPointer(
V, Align, APInt(DL.getIndexTypeSizeInBits(VTy), DL.getTypeStoreSize(Ty)), DL,
CtxI, DT, Visited);
}
void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
const DataLayout &DL,
const FuncInfoTy &FuncInfo) const {
const AttributeSet &Attrs = FuncInfo.getAttributes();
if (Attrs.hasAttribute(OpIdx, Attribute::ZExt))
Arg.Flags.setZExt();
if (Attrs.hasAttribute(OpIdx, Attribute::SExt))
Arg.Flags.setSExt();
if (Attrs.hasAttribute(OpIdx, Attribute::InReg))
Arg.Flags.setInReg();
if (Attrs.hasAttribute(OpIdx, Attribute::StructRet))
Arg.Flags.setSRet();
if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf))
Arg.Flags.setSwiftSelf();
if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError))
Arg.Flags.setSwiftError();
if (Attrs.hasAttribute(OpIdx, Attribute::ByVal))
Arg.Flags.setByVal();
if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca))
Arg.Flags.setInAlloca();
if (Arg.Flags.isByVal() || Arg.Flags.isInAlloca()) {
Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
Arg.Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
// For ByVal, alignment should be passed from FE. BE will guess if
// this info is not there but there are cases it cannot get right.
unsigned FrameAlign;
if (FuncInfo.getParamAlignment(OpIdx))
FrameAlign = FuncInfo.getParamAlignment(OpIdx);
else
FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
Arg.Flags.setByValAlign(FrameAlign);
}
if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
Arg.Flags.setNest();
Arg.Flags.setOrigAlign(DL.getABITypeAlignment(Arg.Ty));
}
/// Test if V is always a pointer to allocated and suitably aligned memory for
/// a simple load or store.
static bool isDereferenceableAndAlignedPointer(
const Value *V, unsigned Align, const DataLayout &DL,
const Instruction *CtxI, const DominatorTree *DT,
const TargetLibraryInfo *TLI, SmallPtrSetImpl<const Value *> &Visited) {
// Note that it is not safe to speculate into a malloc'd region because
// malloc may return null.
// These are obviously ok if aligned.
if (isa<AllocaInst>(V))
return isAligned(V, Align, DL);
// It's not always safe to follow a bitcast, for example:
// bitcast i8* (alloca i8) to i32*
// would result in a 4-byte load from a 1-byte alloca. However,
// if we're casting from a pointer from a type of larger size
// to a type of smaller size (or the same size), and the alignment
// is at least as large as for the resulting pointer type, then
// we can look through the bitcast.
if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V)) {
Type *STy = BC->getSrcTy()->getPointerElementType(),
*DTy = BC->getDestTy()->getPointerElementType();
if (STy->isSized() && DTy->isSized() &&
(DL.getTypeStoreSize(STy) >= DL.getTypeStoreSize(DTy)) &&
(DL.getABITypeAlignment(STy) >= DL.getABITypeAlignment(DTy)))
return isDereferenceableAndAlignedPointer(BC->getOperand(0), Align, DL,
CtxI, DT, TLI, Visited);
}
// Global variables which can't collapse to null are ok.
if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
if (!GV->hasExternalWeakLinkage())
return isAligned(V, Align, DL);
// byval arguments are okay.
if (const Argument *A = dyn_cast<Argument>(V))
if (A->hasByValAttr())
return isAligned(V, Align, DL);
if (isDereferenceableFromAttribute(V, DL, CtxI, DT, TLI))
return isAligned(V, Align, DL);
// For GEPs, determine if the indexing lands within the allocated object.
if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
Type *Ty = GEP->getResultElementType();
const Value *Base = GEP->getPointerOperand();
// Conservatively require that the base pointer be fully dereferenceable
// and aligned.
if (!Visited.insert(Base).second)
return false;
if (!isDereferenceableAndAlignedPointer(Base, Align, DL, CtxI, DT, TLI,
Visited))
return false;
APInt Offset(DL.getPointerTypeSizeInBits(GEP->getType()), 0);
if (!GEP->accumulateConstantOffset(DL, Offset))
return false;
// Check if the load is within the bounds of the underlying object
// and offset is aligned.
uint64_t LoadSize = DL.getTypeStoreSize(Ty);
Type *BaseType = GEP->getSourceElementType();
assert(isPowerOf2_32(Align) && "must be a power of 2!");
return (Offset + LoadSize).ule(DL.getTypeAllocSize(BaseType)) &&
!(Offset & APInt(Offset.getBitWidth(), Align-1));
}
// For gc.relocate, look through relocations
if (const GCRelocateInst *RelocateInst = dyn_cast<GCRelocateInst>(V))
return isDereferenceableAndAlignedPointer(
RelocateInst->getDerivedPtr(), Align, DL, CtxI, DT, TLI, Visited);
if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V))
return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Align, DL,
CtxI, DT, TLI, Visited);
// If we don't know, assume the worst.
return false;
}
/// Check if executing a load of this pointer value cannot trap.
///
/// If DT and ScanFrom are specified this method performs context-sensitive
/// analysis and returns true if it is safe to load immediately before ScanFrom.
///
/// If it is not obviously safe to load from the specified pointer, we do
/// a quick local scan of the basic block containing \c ScanFrom, to determine
/// if the address is already accessed.
///
/// This uses the pointee type to determine how many bytes need to be safe to
/// load from the pointer.
bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align,
const DataLayout &DL,
Instruction *ScanFrom,
const DominatorTree *DT) {
// Zero alignment means that the load has the ABI alignment for the target
if (Align == 0)
Align = DL.getABITypeAlignment(V->getType()->getPointerElementType());
assert(isPowerOf2_32(Align));
// If DT is not specified we can't make context-sensitive query
const Instruction* CtxI = DT ? ScanFrom : nullptr;
if (isDereferenceableAndAlignedPointer(V, Align, DL, CtxI, DT))
return true;
int64_t ByteOffset = 0;
Value *Base = V;
Base = GetPointerBaseWithConstantOffset(V, ByteOffset, DL);
if (ByteOffset < 0) // out of bounds
return false;
Type *BaseType = nullptr;
unsigned BaseAlign = 0;
if (const AllocaInst *AI = dyn_cast<AllocaInst>(Base)) {
// An alloca is safe to load from as load as it is suitably aligned.
BaseType = AI->getAllocatedType();
BaseAlign = AI->getAlignment();
} else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Base)) {
// Global variables are not necessarily safe to load from if they are
// interposed arbitrarily. Their size may change or they may be weak and
// require a test to determine if they were in fact provided.
if (!GV->isInterposable()) {
BaseType = GV->getType()->getElementType();
BaseAlign = GV->getAlignment();
}
}
PointerType *AddrTy = cast<PointerType>(V->getType());
uint64_t LoadSize = DL.getTypeStoreSize(AddrTy->getElementType());
// If we found a base allocated type from either an alloca or global variable,
// try to see if we are definitively within the allocated region. We need to
// know the size of the base type and the loaded type to do anything in this
// case.
if (BaseType && BaseType->isSized()) {
if (BaseAlign == 0)
BaseAlign = DL.getPrefTypeAlignment(BaseType);
if (Align <= BaseAlign) {
// Check if the load is within the bounds of the underlying object.
if (ByteOffset + LoadSize <= DL.getTypeAllocSize(BaseType) &&
((ByteOffset % Align) == 0))
return true;
}
}
if (!ScanFrom)
return false;
// Otherwise, be a little bit aggressive by scanning the local block where we
// want to check to see if the pointer is already being loaded or stored
// from/to. If so, the previous load or store would have already trapped,
// so there is no harm doing an extra load (also, CSE will later eliminate
// the load entirely).
BasicBlock::iterator BBI = ScanFrom->getIterator(),
E = ScanFrom->getParent()->begin();
// We can at least always strip pointer casts even though we can't use the
// base here.
V = V->stripPointerCasts();
while (BBI != E) {
--BBI;
// If we see a free or a call which may write to memory (i.e. which might do
// a free) the pointer could be marked invalid.
if (isa<CallInst>(BBI) && BBI->mayWriteToMemory() &&
!isa<DbgInfoIntrinsic>(BBI))
return false;
Value *AccessedPtr;
unsigned AccessedAlign;
if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) {
AccessedPtr = LI->getPointerOperand();
AccessedAlign = LI->getAlignment();
} else if (StoreInst *SI = dyn_cast<StoreInst>(BBI)) {
AccessedPtr = SI->getPointerOperand();
AccessedAlign = SI->getAlignment();
} else
continue;
Type *AccessedTy = AccessedPtr->getType()->getPointerElementType();
if (AccessedAlign == 0)
AccessedAlign = DL.getABITypeAlignment(AccessedTy);
if (AccessedAlign < Align)
continue;
// Handle trivial cases.
if (AccessedPtr == V)
return true;
if (AreEquivalentAddressValues(AccessedPtr->stripPointerCasts(), V) &&
LoadSize <= DL.getTypeStoreSize(AccessedTy))
return true;
}
return false;
}
void AMDGPUTargetStreamer::emitRuntimeMetadataForKernelArg(const DataLayout &DL,
Type *T, RuntimeMD::KernelArg::Kind Kind,
StringRef BaseTypeName, StringRef TypeName,
StringRef ArgName, StringRef TypeQual, StringRef AccQual) {
auto &S = getStreamer();
// Emit KeyArgBegin.
S.EmitIntValue(RuntimeMD::KeyArgBegin, 1);
// Emit KeyArgSize and KeyArgAlign.
emitRuntimeMDIntValue(RuntimeMD::KeyArgSize,
DL.getTypeAllocSize(T), 4);
emitRuntimeMDIntValue(RuntimeMD::KeyArgAlign,
DL.getABITypeAlignment(T), 4);
if (auto PT = dyn_cast<PointerType>(T)) {
auto ET = PT->getElementType();
if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && ET->isSized())
emitRuntimeMDIntValue(RuntimeMD::KeyArgPointeeAlign,
DL.getABITypeAlignment(ET), 4);
}
// Emit KeyArgTypeName.
if (!TypeName.empty())
emitRuntimeMDStringValue(RuntimeMD::KeyArgTypeName, TypeName);
// Emit KeyArgName.
if (!ArgName.empty())
emitRuntimeMDStringValue(RuntimeMD::KeyArgName, ArgName);
// Emit KeyArgIsVolatile, KeyArgIsRestrict, KeyArgIsConst and KeyArgIsPipe.
SmallVector<StringRef, 1> SplitQ;
TypeQual.split(SplitQ, " ", -1, false /* Drop empty entry */);
for (StringRef KeyName : SplitQ) {
auto Key = StringSwitch<RuntimeMD::Key>(KeyName)
.Case("volatile", RuntimeMD::KeyArgIsVolatile)
.Case("restrict", RuntimeMD::KeyArgIsRestrict)
.Case("const", RuntimeMD::KeyArgIsConst)
.Case("pipe", RuntimeMD::KeyArgIsPipe)
.Default(RuntimeMD::KeyNull);
S.EmitIntValue(Key, 1);
}
// Emit KeyArgKind.
emitRuntimeMDIntValue(RuntimeMD::KeyArgKind, Kind, 1);
// Emit KeyArgValueType.
emitRuntimeMDIntValue(RuntimeMD::KeyArgValueType,
getRuntimeMDValueType(T, BaseTypeName), 2);
// Emit KeyArgAccQual.
if (!AccQual.empty()) {
auto AQ = StringSwitch<RuntimeMD::KernelArg::AccessQualifer>(AccQual)
.Case("read_only", RuntimeMD::KernelArg::ReadOnly)
.Case("write_only", RuntimeMD::KernelArg::WriteOnly)
.Case("read_write", RuntimeMD::KernelArg::ReadWrite)
.Default(RuntimeMD::KernelArg::None);
emitRuntimeMDIntValue(RuntimeMD::KeyArgAccQual, AQ, 1);
}
// Emit KeyArgAddrQual.
if (auto *PT = dyn_cast<PointerType>(T))
emitRuntimeMDIntValue(RuntimeMD::KeyArgAddrQual,
getRuntimeAddrSpace(static_cast<AMDGPUAS::AddressSpaces>(
PT->getAddressSpace())), 1);
// Emit KeyArgEnd
S.EmitIntValue(RuntimeMD::KeyArgEnd, 1);
}
