static KernelArg::Metadata getRuntimeMDForKernelArg(const DataLayout &DL, Type *T, KernelArg::Kind Kind, StringRef BaseTypeName = "", StringRef TypeName = "", StringRef ArgName = "", StringRef TypeQual = "", StringRef AccQual = "") { KernelArg::Metadata Arg; // Set ArgSize and ArgAlign. Arg.Size = DL.getTypeAllocSize(T); Arg.Align = DL.getABITypeAlignment(T); if (auto PT = dyn_cast<PointerType>(T)) { auto ET = PT->getElementType(); if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && ET->isSized()) Arg.PointeeAlign = DL.getABITypeAlignment(ET); } // Set ArgTypeName. Arg.TypeName = TypeName; // Set ArgName. Arg.Name = ArgName; // Set ArgIsVolatile, ArgIsRestrict, ArgIsConst and ArgIsPipe. SmallVector<StringRef, 1> SplitQ; TypeQual.split(SplitQ, " ", -1, false /* Drop empty entry */); for (StringRef KeyName : SplitQ) { auto *P = StringSwitch<uint8_t *>(KeyName) .Case("volatile", &Arg.IsVolatile) .Case("restrict", &Arg.IsRestrict) .Case("const", &Arg.IsConst) .Case("pipe", &Arg.IsPipe) .Default(nullptr); if (P) *P = 1; } // Set ArgKind. Arg.Kind = Kind; // Set ArgValueType. Arg.ValueType = getRuntimeMDValueType(T, BaseTypeName); // Set ArgAccQual. if (!AccQual.empty()) { Arg.AccQual = StringSwitch<KernelArg::AccessQualifer>(AccQual) .Case("read_only", KernelArg::ReadOnly) .Case("write_only", KernelArg::WriteOnly) .Case("read_write", KernelArg::ReadWrite) .Default(KernelArg::AccNone); } // Set ArgAddrQual. if (auto *PT = dyn_cast<PointerType>(T)) { Arg.AddrQual = getRuntimeAddrSpace(static_cast<AMDGPUAS::AddressSpaces>( PT->getAddressSpace())); } return Arg; }
static unsigned findCommonAlignment(const DataLayout &DL, const StoreInst *SI, const LoadInst *LI) { unsigned StoreAlign = SI->getAlignment(); if (!StoreAlign) StoreAlign = DL.getABITypeAlignment(SI->getOperand(0)->getType()); unsigned LoadAlign = LI->getAlignment(); if (!LoadAlign) LoadAlign = DL.getABITypeAlignment(LI->getType()); return std::min(StoreAlign, LoadAlign); }
unsigned Value::getPointerAlignment(const DataLayout &DL) const { assert(getType()->isPointerTy() && "must be pointer"); unsigned Align = 0; if (auto *GO = dyn_cast<GlobalObject>(this)) { // Don't make any assumptions about function pointer alignment. Some // targets use the LSBs to store additional information. if (isa<Function>(GO)) return 0; Align = GO->getAlignment(); if (Align == 0) { if (auto *GVar = dyn_cast<GlobalVariable>(GO)) { Type *ObjectType = GVar->getValueType(); if (ObjectType->isSized()) { // If the object is defined in the current Module, we'll be giving // it the preferred alignment. Otherwise, we have to assume that it // may only have the minimum ABI alignment. if (GVar->isStrongDefinitionForLinker()) Align = DL.getPreferredAlignment(GVar); else Align = DL.getABITypeAlignment(ObjectType); } } } } else if (const Argument *A = dyn_cast<Argument>(this)) { Align = A->getParamAlignment(); if (!Align && A->hasStructRetAttr()) { // An sret parameter has at least the ABI alignment of the return type. Type *EltTy = cast<PointerType>(A->getType())->getElementType(); if (EltTy->isSized()) Align = DL.getABITypeAlignment(EltTy); } } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) { Align = AI->getAlignment(); if (Align == 0) { Type *AllocatedType = AI->getAllocatedType(); if (AllocatedType->isSized()) Align = DL.getPrefTypeAlignment(AllocatedType); } } else if (auto CS = ImmutableCallSite(this)) Align = CS.getAttributes().getRetAlignment(); else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) { ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0)); Align = CI->getLimitedValue(); } return Align; }
bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT, const TargetLibraryInfo *TLI) { // When dereferenceability information is provided by a dereferenceable // attribute, we know exactly how many bytes are dereferenceable. If we can // determine the exact offset to the attributed variable, we can use that // information here. Type *VTy = V->getType(); Type *Ty = VTy->getPointerElementType(); // Require ABI alignment for loads without alignment specification if (Align == 0) Align = DL.getABITypeAlignment(Ty); if (Ty->isSized()) { APInt Offset(DL.getTypeStoreSizeInBits(VTy), 0); const Value *BV = V->stripAndAccumulateInBoundsConstantOffsets(DL, Offset); if (Offset.isNonNegative()) if (isDereferenceableFromAttribute(BV, Offset, Ty, DL, CtxI, DT, TLI) && isAligned(BV, Offset, Align, DL)) return true; } SmallPtrSet<const Value *, 32> Visited; return ::isDereferenceableAndAlignedPointer(V, Align, DL, CtxI, DT, TLI, Visited); }
StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { assert(!ST->isOpaque() && "Cannot get layout of opaque structs"); StructAlignment = 0; StructSize = 0; NumElements = ST->getNumElements(); // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { Type *Ty = ST->getElementType(i); unsigned TyAlign = ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty); // Add padding if necessary to align the data element properly. if ((StructSize & (TyAlign-1)) != 0) StructSize = RoundUpToAlignment(StructSize, TyAlign); // Keep track of maximum alignment constraint. StructAlignment = std::max(TyAlign, StructAlignment); MemberOffsets[i] = StructSize; StructSize += DL.getTypeAllocSize(Ty); // Consume space for this data item } // Empty structures have alignment of 1 byte. if (StructAlignment == 0) StructAlignment = 1; // Add padding to the end of the struct so that it could be put in an array // and all array elements would be aligned correctly. if ((StructSize & (StructAlignment-1)) != 0) StructSize = RoundUpToAlignment(StructSize, StructAlignment); }
static bool isAligned(const Value *Base, APInt Offset, unsigned Align, const DataLayout &DL) { APInt BaseAlign(Offset.getBitWidth(), Base->getPointerAlignment(DL)); if (!BaseAlign) { Type *Ty = Base->getType()->getPointerElementType(); if (!Ty->isSized()) return false; BaseAlign = DL.getABITypeAlignment(Ty); } APInt Alignment(Offset.getBitWidth(), Align); assert(Alignment.isPowerOf2() && "must be a power of 2!"); return BaseAlign.uge(Alignment) && !(Offset & (Alignment-1)); }
/// getPointeeAlignment - Compute the minimum alignment of the value pointed /// to by the given pointer. static unsigned getPointeeAlignment(Value *V, const DataLayout &TD) { if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) if (CE->getOpcode() == Instruction::BitCast || (CE->getOpcode() == Instruction::GetElementPtr && cast<GEPOperator>(CE)->hasAllZeroIndices())) return getPointeeAlignment(CE->getOperand(0), TD); if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) if (!GV->isDeclaration()) return TD.getPreferredAlignment(GV); if (PointerType *PT = dyn_cast<PointerType>(V->getType())) return TD.getABITypeAlignment(PT->getElementType()); return 0; }
// Try to fill in Layout from Ty, returning true on success. Alignment is // the alignment of the vector, or 0 if the ABI default should be used. bool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment, VectorLayout &Layout, const DataLayout &DL) { // Make sure we're dealing with a vector. Layout.VecTy = dyn_cast<VectorType>(Ty); if (!Layout.VecTy) return false; // Check that we're dealing with full-byte elements. Layout.ElemTy = Layout.VecTy->getElementType(); if (DL.getTypeSizeInBits(Layout.ElemTy) != DL.getTypeStoreSizeInBits(Layout.ElemTy)) return false; if (Alignment) Layout.VecAlign = Alignment; else Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy); Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy); return true; }
bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT) { // When dereferenceability information is provided by a dereferenceable // attribute, we know exactly how many bytes are dereferenceable. If we can // determine the exact offset to the attributed variable, we can use that // information here. Type *VTy = V->getType(); Type *Ty = VTy->getPointerElementType(); // Require ABI alignment for loads without alignment specification if (Align == 0) Align = DL.getABITypeAlignment(Ty); if (!Ty->isSized()) return false; SmallPtrSet<const Value *, 32> Visited; return ::isDereferenceableAndAlignedPointer( V, Align, APInt(DL.getIndexTypeSizeInBits(VTy), DL.getTypeStoreSize(Ty)), DL, CtxI, DT, Visited); }
void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const { const AttributeSet &Attrs = FuncInfo.getAttributes(); if (Attrs.hasAttribute(OpIdx, Attribute::ZExt)) Arg.Flags.setZExt(); if (Attrs.hasAttribute(OpIdx, Attribute::SExt)) Arg.Flags.setSExt(); if (Attrs.hasAttribute(OpIdx, Attribute::InReg)) Arg.Flags.setInReg(); if (Attrs.hasAttribute(OpIdx, Attribute::StructRet)) Arg.Flags.setSRet(); if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf)) Arg.Flags.setSwiftSelf(); if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError)) Arg.Flags.setSwiftError(); if (Attrs.hasAttribute(OpIdx, Attribute::ByVal)) Arg.Flags.setByVal(); if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca)) Arg.Flags.setInAlloca(); if (Arg.Flags.isByVal() || Arg.Flags.isInAlloca()) { Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType(); Arg.Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); // For ByVal, alignment should be passed from FE. BE will guess if // this info is not there but there are cases it cannot get right. unsigned FrameAlign; if (FuncInfo.getParamAlignment(OpIdx)) FrameAlign = FuncInfo.getParamAlignment(OpIdx); else FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL); Arg.Flags.setByValAlign(FrameAlign); } if (Attrs.hasAttribute(OpIdx, Attribute::Nest)) Arg.Flags.setNest(); Arg.Flags.setOrigAlign(DL.getABITypeAlignment(Arg.Ty)); }
/// Test if V is always a pointer to allocated and suitably aligned memory for /// a simple load or store. static bool isDereferenceableAndAlignedPointer( const Value *V, unsigned Align, const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT, const TargetLibraryInfo *TLI, SmallPtrSetImpl<const Value *> &Visited) { // Note that it is not safe to speculate into a malloc'd region because // malloc may return null. // These are obviously ok if aligned. if (isa<AllocaInst>(V)) return isAligned(V, Align, DL); // It's not always safe to follow a bitcast, for example: // bitcast i8* (alloca i8) to i32* // would result in a 4-byte load from a 1-byte alloca. However, // if we're casting from a pointer from a type of larger size // to a type of smaller size (or the same size), and the alignment // is at least as large as for the resulting pointer type, then // we can look through the bitcast. if (const BitCastOperator *BC = dyn_cast<BitCastOperator>(V)) { Type *STy = BC->getSrcTy()->getPointerElementType(), *DTy = BC->getDestTy()->getPointerElementType(); if (STy->isSized() && DTy->isSized() && (DL.getTypeStoreSize(STy) >= DL.getTypeStoreSize(DTy)) && (DL.getABITypeAlignment(STy) >= DL.getABITypeAlignment(DTy))) return isDereferenceableAndAlignedPointer(BC->getOperand(0), Align, DL, CtxI, DT, TLI, Visited); } // Global variables which can't collapse to null are ok. if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) if (!GV->hasExternalWeakLinkage()) return isAligned(V, Align, DL); // byval arguments are okay. if (const Argument *A = dyn_cast<Argument>(V)) if (A->hasByValAttr()) return isAligned(V, Align, DL); if (isDereferenceableFromAttribute(V, DL, CtxI, DT, TLI)) return isAligned(V, Align, DL); // For GEPs, determine if the indexing lands within the allocated object. if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { Type *Ty = GEP->getResultElementType(); const Value *Base = GEP->getPointerOperand(); // Conservatively require that the base pointer be fully dereferenceable // and aligned. if (!Visited.insert(Base).second) return false; if (!isDereferenceableAndAlignedPointer(Base, Align, DL, CtxI, DT, TLI, Visited)) return false; APInt Offset(DL.getPointerTypeSizeInBits(GEP->getType()), 0); if (!GEP->accumulateConstantOffset(DL, Offset)) return false; // Check if the load is within the bounds of the underlying object // and offset is aligned. uint64_t LoadSize = DL.getTypeStoreSize(Ty); Type *BaseType = GEP->getSourceElementType(); assert(isPowerOf2_32(Align) && "must be a power of 2!"); return (Offset + LoadSize).ule(DL.getTypeAllocSize(BaseType)) && !(Offset & APInt(Offset.getBitWidth(), Align-1)); } // For gc.relocate, look through relocations if (const GCRelocateInst *RelocateInst = dyn_cast<GCRelocateInst>(V)) return isDereferenceableAndAlignedPointer( RelocateInst->getDerivedPtr(), Align, DL, CtxI, DT, TLI, Visited); if (const AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(V)) return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Align, DL, CtxI, DT, TLI, Visited); // If we don't know, assume the worst. return false; }
/// Check if executing a load of this pointer value cannot trap. /// /// If DT and ScanFrom are specified this method performs context-sensitive /// analysis and returns true if it is safe to load immediately before ScanFrom. /// /// If it is not obviously safe to load from the specified pointer, we do /// a quick local scan of the basic block containing \c ScanFrom, to determine /// if the address is already accessed. /// /// This uses the pointee type to determine how many bytes need to be safe to /// load from the pointer. bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, const DataLayout &DL, Instruction *ScanFrom, const DominatorTree *DT) { // Zero alignment means that the load has the ABI alignment for the target if (Align == 0) Align = DL.getABITypeAlignment(V->getType()->getPointerElementType()); assert(isPowerOf2_32(Align)); // If DT is not specified we can't make context-sensitive query const Instruction* CtxI = DT ? ScanFrom : nullptr; if (isDereferenceableAndAlignedPointer(V, Align, DL, CtxI, DT)) return true; int64_t ByteOffset = 0; Value *Base = V; Base = GetPointerBaseWithConstantOffset(V, ByteOffset, DL); if (ByteOffset < 0) // out of bounds return false; Type *BaseType = nullptr; unsigned BaseAlign = 0; if (const AllocaInst *AI = dyn_cast<AllocaInst>(Base)) { // An alloca is safe to load from as load as it is suitably aligned. BaseType = AI->getAllocatedType(); BaseAlign = AI->getAlignment(); } else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Base)) { // Global variables are not necessarily safe to load from if they are // interposed arbitrarily. Their size may change or they may be weak and // require a test to determine if they were in fact provided. if (!GV->isInterposable()) { BaseType = GV->getType()->getElementType(); BaseAlign = GV->getAlignment(); } } PointerType *AddrTy = cast<PointerType>(V->getType()); uint64_t LoadSize = DL.getTypeStoreSize(AddrTy->getElementType()); // If we found a base allocated type from either an alloca or global variable, // try to see if we are definitively within the allocated region. We need to // know the size of the base type and the loaded type to do anything in this // case. if (BaseType && BaseType->isSized()) { if (BaseAlign == 0) BaseAlign = DL.getPrefTypeAlignment(BaseType); if (Align <= BaseAlign) { // Check if the load is within the bounds of the underlying object. if (ByteOffset + LoadSize <= DL.getTypeAllocSize(BaseType) && ((ByteOffset % Align) == 0)) return true; } } if (!ScanFrom) return false; // Otherwise, be a little bit aggressive by scanning the local block where we // want to check to see if the pointer is already being loaded or stored // from/to. If so, the previous load or store would have already trapped, // so there is no harm doing an extra load (also, CSE will later eliminate // the load entirely). BasicBlock::iterator BBI = ScanFrom->getIterator(), E = ScanFrom->getParent()->begin(); // We can at least always strip pointer casts even though we can't use the // base here. V = V->stripPointerCasts(); while (BBI != E) { --BBI; // If we see a free or a call which may write to memory (i.e. which might do // a free) the pointer could be marked invalid. if (isa<CallInst>(BBI) && BBI->mayWriteToMemory() && !isa<DbgInfoIntrinsic>(BBI)) return false; Value *AccessedPtr; unsigned AccessedAlign; if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) { AccessedPtr = LI->getPointerOperand(); AccessedAlign = LI->getAlignment(); } else if (StoreInst *SI = dyn_cast<StoreInst>(BBI)) { AccessedPtr = SI->getPointerOperand(); AccessedAlign = SI->getAlignment(); } else continue; Type *AccessedTy = AccessedPtr->getType()->getPointerElementType(); if (AccessedAlign == 0) AccessedAlign = DL.getABITypeAlignment(AccessedTy); if (AccessedAlign < Align) continue; // Handle trivial cases. if (AccessedPtr == V) return true; if (AreEquivalentAddressValues(AccessedPtr->stripPointerCasts(), V) && LoadSize <= DL.getTypeStoreSize(AccessedTy)) return true; } return false; }
void AMDGPUTargetStreamer::emitRuntimeMetadataForKernelArg(const DataLayout &DL, Type *T, RuntimeMD::KernelArg::Kind Kind, StringRef BaseTypeName, StringRef TypeName, StringRef ArgName, StringRef TypeQual, StringRef AccQual) { auto &S = getStreamer(); // Emit KeyArgBegin. S.EmitIntValue(RuntimeMD::KeyArgBegin, 1); // Emit KeyArgSize and KeyArgAlign. emitRuntimeMDIntValue(RuntimeMD::KeyArgSize, DL.getTypeAllocSize(T), 4); emitRuntimeMDIntValue(RuntimeMD::KeyArgAlign, DL.getABITypeAlignment(T), 4); if (auto PT = dyn_cast<PointerType>(T)) { auto ET = PT->getElementType(); if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && ET->isSized()) emitRuntimeMDIntValue(RuntimeMD::KeyArgPointeeAlign, DL.getABITypeAlignment(ET), 4); } // Emit KeyArgTypeName. if (!TypeName.empty()) emitRuntimeMDStringValue(RuntimeMD::KeyArgTypeName, TypeName); // Emit KeyArgName. if (!ArgName.empty()) emitRuntimeMDStringValue(RuntimeMD::KeyArgName, ArgName); // Emit KeyArgIsVolatile, KeyArgIsRestrict, KeyArgIsConst and KeyArgIsPipe. SmallVector<StringRef, 1> SplitQ; TypeQual.split(SplitQ, " ", -1, false /* Drop empty entry */); for (StringRef KeyName : SplitQ) { auto Key = StringSwitch<RuntimeMD::Key>(KeyName) .Case("volatile", RuntimeMD::KeyArgIsVolatile) .Case("restrict", RuntimeMD::KeyArgIsRestrict) .Case("const", RuntimeMD::KeyArgIsConst) .Case("pipe", RuntimeMD::KeyArgIsPipe) .Default(RuntimeMD::KeyNull); S.EmitIntValue(Key, 1); } // Emit KeyArgKind. emitRuntimeMDIntValue(RuntimeMD::KeyArgKind, Kind, 1); // Emit KeyArgValueType. emitRuntimeMDIntValue(RuntimeMD::KeyArgValueType, getRuntimeMDValueType(T, BaseTypeName), 2); // Emit KeyArgAccQual. if (!AccQual.empty()) { auto AQ = StringSwitch<RuntimeMD::KernelArg::AccessQualifer>(AccQual) .Case("read_only", RuntimeMD::KernelArg::ReadOnly) .Case("write_only", RuntimeMD::KernelArg::WriteOnly) .Case("read_write", RuntimeMD::KernelArg::ReadWrite) .Default(RuntimeMD::KernelArg::None); emitRuntimeMDIntValue(RuntimeMD::KeyArgAccQual, AQ, 1); } // Emit KeyArgAddrQual. if (auto *PT = dyn_cast<PointerType>(T)) emitRuntimeMDIntValue(RuntimeMD::KeyArgAddrQual, getRuntimeAddrSpace(static_cast<AMDGPUAS::AddressSpaces>( PT->getAddressSpace())), 1); // Emit KeyArgEnd S.EmitIntValue(RuntimeMD::KeyArgEnd, 1); }