static void appendToGlobalArray(const char *Array, Module &M, Function *F, int Priority) { IRBuilder<> IRB(M.getContext()); FunctionType *FnTy = FunctionType::get(IRB.getVoidTy(), false); // Get the current set of static global constructors and add the new ctor // to the list. SmallVector<Constant *, 16> CurrentCtors; StructType *EltTy; if (GlobalVariable *GVCtor = M.getNamedGlobal(Array)) { // If there is a global_ctors array, use the existing struct type, which can // have 2 or 3 fields. ArrayType *ATy = cast<ArrayType>(GVCtor->getType()->getElementType()); EltTy = cast<StructType>(ATy->getElementType()); if (Constant *Init = GVCtor->getInitializer()) { unsigned n = Init->getNumOperands(); CurrentCtors.reserve(n + 1); for (unsigned i = 0; i != n; ++i) CurrentCtors.push_back(cast<Constant>(Init->getOperand(i))); } GVCtor->eraseFromParent(); } else { // Use a simple two-field struct if there isn't one already. EltTy = StructType::get(IRB.getInt32Ty(), PointerType::getUnqual(FnTy), nullptr); } // Build a 2 or 3 field global_ctor entry. We don't take a comdat key. Constant *CSVals[3]; CSVals[0] = IRB.getInt32(Priority); CSVals[1] = F; // FIXME: Drop support for the two element form in LLVM 4.0. if (EltTy->getNumElements() >= 3) CSVals[2] = llvm::Constant::getNullValue(IRB.getInt8PtrTy()); Constant *RuntimeCtorInit = ConstantStruct::get(EltTy, makeArrayRef(CSVals, EltTy->getNumElements())); CurrentCtors.push_back(RuntimeCtorInit); // Create a new initializer. ArrayType *AT = ArrayType::get(EltTy, CurrentCtors.size()); Constant *NewInit = ConstantArray::get(AT, CurrentCtors); // Create the new global variable and replace all uses of // the old global variable with the new one. (void)new GlobalVariable(M, NewInit->getType(), false, GlobalValue::AppendingLinkage, NewInit, Array); }
Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) { switch (Inst->getIntrinsicID()) { default: return nullptr; case Intrinsic::aarch64_neon_st2: case Intrinsic::aarch64_neon_st3: case Intrinsic::aarch64_neon_st4: { // Create a struct type StructType *ST = dyn_cast<StructType>(ExpectedType); if (!ST) return nullptr; unsigned NumElts = Inst->getNumArgOperands() - 1; if (ST->getNumElements() != NumElts) return nullptr; for (unsigned i = 0, e = NumElts; i != e; ++i) { if (Inst->getArgOperand(i)->getType() != ST->getElementType(i)) return nullptr; } Value *Res = UndefValue::get(ExpectedType); IRBuilder<> Builder(Inst); for (unsigned i = 0, e = NumElts; i != e; ++i) { Value *L = Inst->getArgOperand(i); Res = Builder.CreateInsertValue(Res, L, i); } return Res; } case Intrinsic::aarch64_neon_ld2: case Intrinsic::aarch64_neon_ld3: case Intrinsic::aarch64_neon_ld4: if (Inst->getType() == ExpectedType) return Inst; return nullptr; } }
/// \brief Checks if a type could have padding bytes. bool ArgPromotion::isDenselyPacked(Type *type) { // There is no size information, so be conservative. if (!type->isSized()) return false; // If the alloc size is not equal to the storage size, then there are padding // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128. if (!DL || DL->getTypeSizeInBits(type) != DL->getTypeAllocSizeInBits(type)) return false; if (!isa<CompositeType>(type)) return true; // For homogenous sequential types, check for padding within members. if (SequentialType *seqTy = dyn_cast<SequentialType>(type)) return isa<PointerType>(seqTy) || isDenselyPacked(seqTy->getElementType()); // Check for padding within and between elements of a struct. StructType *StructTy = cast<StructType>(type); const StructLayout *Layout = DL->getStructLayout(StructTy); uint64_t StartPos = 0; for (unsigned i = 0, E = StructTy->getNumElements(); i < E; ++i) { Type *ElTy = StructTy->getElementType(i); if (!isDenselyPacked(ElTy)) return false; if (StartPos != Layout->getElementOffsetInBits(i)) return false; StartPos += DL->getTypeAllocSizeInBits(ElTy); } return true; }
static void SplitUpPHINode(PHINode *Phi) { StructType *STy = cast<StructType>(Phi->getType()); Value *NewStruct = UndefValue::get(STy); Instruction *NewStructInsertPt = Phi->getParent()->getFirstInsertionPt(); // Create a separate PHINode for each struct field. for (unsigned Index = 0; Index < STy->getNumElements(); ++Index) { SmallVector<unsigned, 1> EVIndexes; EVIndexes.push_back(Index); PHINode *NewPhi = PHINode::Create( STy->getElementType(Index), Phi->getNumIncomingValues(), Phi->getName() + ".index", Phi); CopyDebug(NewPhi, Phi); for (unsigned PhiIndex = 0; PhiIndex < Phi->getNumIncomingValues(); ++PhiIndex) { BasicBlock *IncomingBB = Phi->getIncomingBlock(PhiIndex); Value *EV = CopyDebug( ExtractValueInst::Create( Phi->getIncomingValue(PhiIndex), EVIndexes, Phi->getName() + ".extract", IncomingBB->getTerminator()), Phi); NewPhi->addIncoming(EV, IncomingBB); } // Reconstruct the original struct value. NewStruct = CopyDebug( InsertValueInst::Create(NewStruct, NewPhi, EVIndexes, Phi->getName() + ".insert", NewStructInsertPt), Phi); } Phi->replaceAllUsesWith(NewStruct); Phi->eraseFromParent(); }
static bool SplitUpStore(StoreInst *Store, const DataLayout *DL) { StructType *STy = cast<StructType>(Store->getValueOperand()->getType()); bool NeedsAnotherPass = false; // Create a separate store instruction for each struct field. for (unsigned Index = 0; Index < STy->getNumElements(); ++Index) { SmallVector<Value *, 2> Indexes; Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, 0))); Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, Index))); Value *GEP = CopyDebug(GetElementPtrInst::Create(STy, Store->getPointerOperand(), Indexes, Store->getPointerOperand()->getName() + ".index", Store), Store); NeedsAnotherPass = NeedsAnotherPass || DoAnotherPass(GEP->getType()->getContainedType(0)); SmallVector<unsigned, 1> EVIndexes; EVIndexes.push_back(Index); Value *Field = ExtractValueInst::Create(Store->getValueOperand(), EVIndexes, "", Store); StoreInst *NewStore = new StoreInst(Field, GEP, Store); ProcessLoadOrStoreAttrs(NewStore, Store, STy, Index, DL); } Store->eraseFromParent(); return NeedsAnotherPass; }
static void SplitUpLoad(LoadInst *Load) { StructType *STy = cast<StructType>(Load->getType()); Value *NewStruct = UndefValue::get(STy); // Create a separate load instruction for each struct field. for (unsigned Index = 0; Index < STy->getNumElements(); ++Index) { SmallVector<Value *, 2> Indexes; Indexes.push_back(ConstantInt::get(Load->getContext(), APInt(32, 0))); Indexes.push_back(ConstantInt::get(Load->getContext(), APInt(32, Index))); Value *GEP = CopyDebug( GetElementPtrInst::Create(Load->getPointerOperand(), Indexes, Load->getName() + ".index", Load), Load); LoadInst *NewLoad = new LoadInst(GEP, Load->getName() + ".field", Load); ProcessLoadOrStoreAttrs(NewLoad, Load); // Reconstruct the struct value. SmallVector<unsigned, 1> EVIndexes; EVIndexes.push_back(Index); NewStruct = CopyDebug( InsertValueInst::Create(NewStruct, NewLoad, EVIndexes, Load->getName() + ".insert", Load), Load); } Load->replaceAllUsesWith(NewStruct); Load->eraseFromParent(); }
bool AllocaMerging::areTypesEquivalent(const cheerp::TypeSupport& types, cheerp::PointerAnalyzer& PA, Type* a, Type* b) { //TODO: Integer types may be equivalent as well if(a==b) return true; else if(a->isPointerTy() && b->isPointerTy()) return true; else if(a->isFloatingPointTy() && b->isFloatingPointTy()) return true; else if(a->isArrayTy() && b->isArrayTy()) { return cast<ArrayType>(a)->getNumElements()==cast<ArrayType>(b)->getNumElements() && areTypesEquivalent(types, PA, a->getArrayElementType(), b->getArrayElementType()); } else if(a->isStructTy() && b->isStructTy()) { // TODO: Byte layout structs with the same size are equivalent if(cast<StructType>(a)->hasByteLayout() || cast<StructType>(b)->hasByteLayout()) return false; StructType* stA = cast<StructType>(a); StructType* stB = cast<StructType>(b); if(stA->getNumElements() != stB->getNumElements()) return false; for(uint32_t i=0;i<stA->getNumElements();i++) { Type* elementA = stA->getElementType(i); Type* elementB = stB->getElementType(i); // The types needs to have consistent wrapper arrays if(types.useWrapperArrayForMember(PA, stA, i) ^ types.useWrapperArrayForMember(PA, stB, i)) return false; if(!areTypesEquivalent(types, PA, elementA, elementB)) return false; } return true; } else return false; }
bool EfficiencySanitizer::instrumentGetElementPtr(Instruction *I, Module &M) { GetElementPtrInst *GepInst = dyn_cast<GetElementPtrInst>(I); bool Res = false; if (GepInst == nullptr || GepInst->getNumIndices() == 1) { ++NumIgnoredGEPs; return false; } Type *SourceTy = GepInst->getSourceElementType(); StructType *StructTy = nullptr; ConstantInt *Idx; // Check if GEP calculates address from a struct array. if (isa<StructType>(SourceTy)) { StructTy = cast<StructType>(SourceTy); Idx = dyn_cast<ConstantInt>(GepInst->getOperand(1)); if ((Idx == nullptr || Idx->getSExtValue() != 0) && !shouldIgnoreStructType(StructTy) && StructTyMap.count(StructTy) != 0) Res |= insertCounterUpdate(I, StructTy, getArrayCounterIdx(StructTy)); } // Iterate all (except the first and the last) idx within each GEP instruction // for possible nested struct field address calculation. for (unsigned i = 1; i < GepInst->getNumIndices(); ++i) { SmallVector<Value *, 8> IdxVec(GepInst->idx_begin(), GepInst->idx_begin() + i); Type *Ty = GetElementPtrInst::getIndexedType(SourceTy, IdxVec); unsigned CounterIdx = 0; if (isa<ArrayType>(Ty)) { ArrayType *ArrayTy = cast<ArrayType>(Ty); StructTy = dyn_cast<StructType>(ArrayTy->getElementType()); if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0) continue; // The last counter for struct array access. CounterIdx = getArrayCounterIdx(StructTy); } else if (isa<StructType>(Ty)) { StructTy = cast<StructType>(Ty); if (shouldIgnoreStructType(StructTy) || StructTyMap.count(StructTy) == 0) continue; // Get the StructTy's subfield index. Idx = cast<ConstantInt>(GepInst->getOperand(i+1)); assert(Idx->getSExtValue() >= 0 && Idx->getSExtValue() < StructTy->getNumElements()); CounterIdx = getFieldCounterIdx(StructTy) + Idx->getSExtValue(); } Res |= insertCounterUpdate(I, StructTy, CounterIdx); } if (Res) ++NumInstrumentedGEPs; else ++NumIgnoredGEPs; return Res; }
/// Verify - Verify that the specified constraint string is reasonable for the /// specified function type, and otherwise validate the constraint string. bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) { if (Ty->isVarArg()) return false; ConstraintInfoVector Constraints = ParseConstraints(ConstStr); // Error parsing constraints. if (Constraints.empty() && !ConstStr.empty()) return false; unsigned NumOutputs = 0, NumInputs = 0, NumClobbers = 0; unsigned NumIndirect = 0; for (unsigned i = 0, e = Constraints.size(); i != e; ++i) { switch (Constraints[i].Type) { case InlineAsm::isOutput: if ((NumInputs-NumIndirect) != 0 || NumClobbers != 0) return false; // outputs before inputs and clobbers. if (!Constraints[i].isIndirect) { ++NumOutputs; break; } ++NumIndirect; // FALLTHROUGH for Indirect Outputs. case InlineAsm::isInput: if (NumClobbers) return false; // inputs before clobbers. ++NumInputs; break; case InlineAsm::isClobber: ++NumClobbers; break; } } switch (NumOutputs) { case 0: if (!Ty->getReturnType()->isVoidTy()) return false; break; case 1: if (Ty->getReturnType()->isStructTy()) return false; break; default: StructType *STy = dyn_cast<StructType>(Ty->getReturnType()); if (!STy || STy->getNumElements() != NumOutputs) return false; break; } if (Ty->getNumParams() != NumInputs) return false; return true; }
static void SplitUpStore(StoreInst *Store) { StructType *STy = cast<StructType>(Store->getValueOperand()->getType()); // Create a separate store instruction for each struct field. for (unsigned Index = 0; Index < STy->getNumElements(); ++Index) { SmallVector<Value *, 2> Indexes; Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, 0))); Indexes.push_back(ConstantInt::get(Store->getContext(), APInt(32, Index))); Value *GEP = CopyDebug(GetElementPtrInst::Create( Store->getPointerOperand(), Indexes, Store->getPointerOperand()->getName() + ".index", Store), Store); SmallVector<unsigned, 1> EVIndexes; EVIndexes.push_back(Index); Value *Field = ExtractValueInst::Create(Store->getValueOperand(), EVIndexes, "", Store); StoreInst *NewStore = new StoreInst(Field, GEP, Store); ProcessLoadOrStoreAttrs(NewStore, Store); } Store->eraseFromParent(); }
/// linkDefinedTypeBodies - Produce a body for an opaque type in the dest /// module from a type definition in the source module. void TypeMapTy::linkDefinedTypeBodies() { SmallVector<Type*, 16> Elements; SmallString<16> TmpName; // Note that processing entries in this loop (calling 'get') can add new // entries to the SrcDefinitionsToResolve vector. while (!SrcDefinitionsToResolve.empty()) { StructType *SrcSTy = SrcDefinitionsToResolve.pop_back_val(); StructType *DstSTy = cast<StructType>(MappedTypes[SrcSTy]); // TypeMap is a many-to-one mapping, if there were multiple types that // provide a body for DstSTy then previous iterations of this loop may have // already handled it. Just ignore this case. if (!DstSTy->isOpaque()) continue; assert(!SrcSTy->isOpaque() && "Not resolving a definition?"); // Map the body of the source type over to a new body for the dest type. Elements.resize(SrcSTy->getNumElements()); for (unsigned i = 0, e = Elements.size(); i != e; ++i) Elements[i] = getImpl(SrcSTy->getElementType(i)); DstSTy->setBody(Elements, SrcSTy->isPacked()); // If DstSTy has no name or has a longer name than STy, then viciously steal // STy's name. if (!SrcSTy->hasName()) continue; StringRef SrcName = SrcSTy->getName(); if (!DstSTy->hasName() || DstSTy->getName().size() > SrcName.size()) { TmpName.insert(TmpName.end(), SrcName.begin(), SrcName.end()); SrcSTy->setName(""); DstSTy->setName(TmpName.str()); TmpName.clear(); } } DstResolvedOpaqueTypes.clear(); }
static bool SplitUpSelect(SelectInst *Select) { StructType *STy = cast<StructType>(Select->getType()); Value *NewStruct = UndefValue::get(STy); bool NeedsAnotherPass = false; // Create a separate SelectInst for each struct field. for (unsigned Index = 0; Index < STy->getNumElements(); ++Index) { SmallVector<unsigned, 1> EVIndexes; EVIndexes.push_back(Index); Value *TrueVal = CopyDebug( ExtractValueInst::Create(Select->getTrueValue(), EVIndexes, Select->getName() + ".extract", Select), Select); Value *FalseVal = CopyDebug( ExtractValueInst::Create(Select->getFalseValue(), EVIndexes, Select->getName() + ".extract", Select), Select); Value *NewSelect = CopyDebug(SelectInst::Create(Select->getCondition(), TrueVal, FalseVal, Select->getName() + ".index", Select), Select); NeedsAnotherPass = NeedsAnotherPass || DoAnotherPass(NewSelect); // Reconstruct the original struct value. NewStruct = CopyDebug( InsertValueInst::Create(NewStruct, NewSelect, EVIndexes, Select->getName() + ".insert", Select), Select); } Select->replaceAllUsesWith(NewStruct); Select->eraseFromParent(); return NeedsAnotherPass; }
/// DoPromotion - This method actually performs the promotion of the specified /// arguments, and returns the new function. At this point, we know that it's /// safe to do so. CallGraphNode *ArgPromotion::DoPromotion(Function *F, SmallPtrSetImpl<Argument*> &ArgsToPromote, SmallPtrSetImpl<Argument*> &ByValArgsToTransform) { // Start by computing a new prototype for the function, which is the same as // the old function, but has modified arguments. FunctionType *FTy = F->getFunctionType(); std::vector<Type*> Params; typedef std::set<IndicesVector> ScalarizeTable; // ScalarizedElements - If we are promoting a pointer that has elements // accessed out of it, keep track of which elements are accessed so that we // can add one argument for each. // // Arguments that are directly loaded will have a zero element value here, to // handle cases where there are both a direct load and GEP accesses. // std::map<Argument*, ScalarizeTable> ScalarizedElements; // OriginalLoads - Keep track of a representative load instruction from the // original function so that we can tell the alias analysis implementation // what the new GEP/Load instructions we are inserting look like. // We need to keep the original loads for each argument and the elements // of the argument that are accessed. std::map<std::pair<Argument*, IndicesVector>, LoadInst*> OriginalLoads; // Attribute - Keep track of the parameter attributes for the arguments // that we are *not* promoting. For the ones that we do promote, the parameter // attributes are lost SmallVector<AttributeSet, 8> AttributesVec; const AttributeSet &PAL = F->getAttributes(); // Add any return attributes. if (PAL.hasAttributes(AttributeSet::ReturnIndex)) AttributesVec.push_back(AttributeSet::get(F->getContext(), PAL.getRetAttributes())); // First, determine the new argument list unsigned ArgIndex = 1; for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I, ++ArgIndex) { if (ByValArgsToTransform.count(I)) { // Simple byval argument? Just add all the struct element types. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); StructType *STy = cast<StructType>(AgTy); for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) Params.push_back(STy->getElementType(i)); ++NumByValArgsPromoted; } else if (!ArgsToPromote.count(I)) { // Unchanged argument Params.push_back(I->getType()); AttributeSet attrs = PAL.getParamAttributes(ArgIndex); if (attrs.hasAttributes(ArgIndex)) { AttrBuilder B(attrs, ArgIndex); AttributesVec. push_back(AttributeSet::get(F->getContext(), Params.size(), B)); } } else if (I->use_empty()) { // Dead argument (which are always marked as promotable) ++NumArgumentsDead; } else { // Okay, this is being promoted. This means that the only uses are loads // or GEPs which are only used by loads // In this table, we will track which indices are loaded from the argument // (where direct loads are tracked as no indices). ScalarizeTable &ArgIndices = ScalarizedElements[I]; for (User *U : I->users()) { Instruction *UI = cast<Instruction>(U); assert(isa<LoadInst>(UI) || isa<GetElementPtrInst>(UI)); IndicesVector Indices; Indices.reserve(UI->getNumOperands() - 1); // Since loads will only have a single operand, and GEPs only a single // non-index operand, this will record direct loads without any indices, // and gep+loads with the GEP indices. for (User::op_iterator II = UI->op_begin() + 1, IE = UI->op_end(); II != IE; ++II) Indices.push_back(cast<ConstantInt>(*II)->getSExtValue()); // GEPs with a single 0 index can be merged with direct loads if (Indices.size() == 1 && Indices.front() == 0) Indices.clear(); ArgIndices.insert(Indices); LoadInst *OrigLoad; if (LoadInst *L = dyn_cast<LoadInst>(UI)) OrigLoad = L; else // Take any load, we will use it only to update Alias Analysis OrigLoad = cast<LoadInst>(UI->user_back()); OriginalLoads[std::make_pair(I, Indices)] = OrigLoad; } // Add a parameter to the function for each element passed in. for (ScalarizeTable::iterator SI = ArgIndices.begin(), E = ArgIndices.end(); SI != E; ++SI) { // not allowed to dereference ->begin() if size() is 0 Params.push_back(GetElementPtrInst::getIndexedType(I->getType(), *SI)); assert(Params.back()); } if (ArgIndices.size() == 1 && ArgIndices.begin()->empty()) ++NumArgumentsPromoted; else ++NumAggregatesPromoted; } } // Add any function attributes. if (PAL.hasAttributes(AttributeSet::FunctionIndex)) AttributesVec.push_back(AttributeSet::get(FTy->getContext(), PAL.getFnAttributes())); Type *RetTy = FTy->getReturnType(); // Construct the new function type using the new arguments. FunctionType *NFTy = FunctionType::get(RetTy, Params, FTy->isVarArg()); // Create the new function body and insert it into the module. Function *NF = Function::Create(NFTy, F->getLinkage(), F->getName()); NF->copyAttributesFrom(F); // Patch the pointer to LLVM function in debug info descriptor. auto DI = FunctionDIs.find(F); if (DI != FunctionDIs.end()) { DISubprogram SP = DI->second; SP.replaceFunction(NF); // Ensure the map is updated so it can be reused on subsequent argument // promotions of the same function. FunctionDIs.erase(DI); FunctionDIs[NF] = SP; } DEBUG(dbgs() << "ARG PROMOTION: Promoting to:" << *NF << "\n" << "From: " << *F); // Recompute the parameter attributes list based on the new arguments for // the function. NF->setAttributes(AttributeSet::get(F->getContext(), AttributesVec)); AttributesVec.clear(); F->getParent()->getFunctionList().insert(F, NF); NF->takeName(F); // Get the alias analysis information that we need to update to reflect our // changes. AliasAnalysis &AA = getAnalysis<AliasAnalysis>(); // Get the callgraph information that we need to update to reflect our // changes. CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph(); // Get a new callgraph node for NF. CallGraphNode *NF_CGN = CG.getOrInsertFunction(NF); // Loop over all of the callers of the function, transforming the call sites // to pass in the loaded pointers. // SmallVector<Value*, 16> Args; while (!F->use_empty()) { CallSite CS(F->user_back()); assert(CS.getCalledFunction() == F); Instruction *Call = CS.getInstruction(); const AttributeSet &CallPAL = CS.getAttributes(); // Add any return attributes. if (CallPAL.hasAttributes(AttributeSet::ReturnIndex)) AttributesVec.push_back(AttributeSet::get(F->getContext(), CallPAL.getRetAttributes())); // Loop over the operands, inserting GEP and loads in the caller as // appropriate. CallSite::arg_iterator AI = CS.arg_begin(); ArgIndex = 1; for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I, ++AI, ++ArgIndex) if (!ArgsToPromote.count(I) && !ByValArgsToTransform.count(I)) { Args.push_back(*AI); // Unmodified argument if (CallPAL.hasAttributes(ArgIndex)) { AttrBuilder B(CallPAL, ArgIndex); AttributesVec. push_back(AttributeSet::get(F->getContext(), Args.size(), B)); } } else if (ByValArgsToTransform.count(I)) { // Emit a GEP and load for each element of the struct. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); StructType *STy = cast<StructType>(AgTy); Value *Idxs[2] = { ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr }; for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i); Value *Idx = GetElementPtrInst::Create(*AI, Idxs, (*AI)->getName()+"."+utostr(i), Call); // TODO: Tell AA about the new values? Args.push_back(new LoadInst(Idx, Idx->getName()+".val", Call)); } } else if (!I->use_empty()) { // Non-dead argument: insert GEPs and loads as appropriate. ScalarizeTable &ArgIndices = ScalarizedElements[I]; // Store the Value* version of the indices in here, but declare it now // for reuse. std::vector<Value*> Ops; for (ScalarizeTable::iterator SI = ArgIndices.begin(), E = ArgIndices.end(); SI != E; ++SI) { Value *V = *AI; LoadInst *OrigLoad = OriginalLoads[std::make_pair(I, *SI)]; if (!SI->empty()) { Ops.reserve(SI->size()); Type *ElTy = V->getType(); for (IndicesVector::const_iterator II = SI->begin(), IE = SI->end(); II != IE; ++II) { // Use i32 to index structs, and i64 for others (pointers/arrays). // This satisfies GEP constraints. Type *IdxTy = (ElTy->isStructTy() ? Type::getInt32Ty(F->getContext()) : Type::getInt64Ty(F->getContext())); Ops.push_back(ConstantInt::get(IdxTy, *II)); // Keep track of the type we're currently indexing. ElTy = cast<CompositeType>(ElTy)->getTypeAtIndex(*II); } // And create a GEP to extract those indices. V = GetElementPtrInst::Create(V, Ops, V->getName()+".idx", Call); Ops.clear(); AA.copyValue(OrigLoad->getOperand(0), V); } // Since we're replacing a load make sure we take the alignment // of the previous load. LoadInst *newLoad = new LoadInst(V, V->getName()+".val", Call); newLoad->setAlignment(OrigLoad->getAlignment()); // Transfer the AA info too. AAMDNodes AAInfo; OrigLoad->getAAMetadata(AAInfo); newLoad->setAAMetadata(AAInfo); Args.push_back(newLoad); AA.copyValue(OrigLoad, Args.back()); } } // Push any varargs arguments on the list. for (; AI != CS.arg_end(); ++AI, ++ArgIndex) { Args.push_back(*AI); if (CallPAL.hasAttributes(ArgIndex)) { AttrBuilder B(CallPAL, ArgIndex); AttributesVec. push_back(AttributeSet::get(F->getContext(), Args.size(), B)); } } // Add any function attributes. if (CallPAL.hasAttributes(AttributeSet::FunctionIndex)) AttributesVec.push_back(AttributeSet::get(Call->getContext(), CallPAL.getFnAttributes())); Instruction *New; if (InvokeInst *II = dyn_cast<InvokeInst>(Call)) { New = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(), Args, "", Call); cast<InvokeInst>(New)->setCallingConv(CS.getCallingConv()); cast<InvokeInst>(New)->setAttributes(AttributeSet::get(II->getContext(), AttributesVec)); } else { New = CallInst::Create(NF, Args, "", Call); cast<CallInst>(New)->setCallingConv(CS.getCallingConv()); cast<CallInst>(New)->setAttributes(AttributeSet::get(New->getContext(), AttributesVec)); if (cast<CallInst>(Call)->isTailCall()) cast<CallInst>(New)->setTailCall(); } New->setDebugLoc(Call->getDebugLoc()); Args.clear(); AttributesVec.clear(); // Update the alias analysis implementation to know that we are replacing // the old call with a new one. AA.replaceWithNewValue(Call, New); // Update the callgraph to know that the callsite has been transformed. CallGraphNode *CalleeNode = CG[Call->getParent()->getParent()]; CalleeNode->replaceCallEdge(Call, New, NF_CGN); if (!Call->use_empty()) { Call->replaceAllUsesWith(New); New->takeName(Call); } // Finally, remove the old call from the program, reducing the use-count of // F. Call->eraseFromParent(); } // Since we have now created the new function, splice the body of the old // function right into the new function, leaving the old rotting hulk of the // function empty. NF->getBasicBlockList().splice(NF->begin(), F->getBasicBlockList()); // Loop over the argument list, transferring uses of the old arguments over to // the new arguments, also transferring over the names as well. // for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(), I2 = NF->arg_begin(); I != E; ++I) { if (!ArgsToPromote.count(I) && !ByValArgsToTransform.count(I)) { // If this is an unmodified argument, move the name and users over to the // new version. I->replaceAllUsesWith(I2); I2->takeName(I); AA.replaceWithNewValue(I, I2); ++I2; continue; } if (ByValArgsToTransform.count(I)) { // In the callee, we create an alloca, and store each of the new incoming // arguments into the alloca. Instruction *InsertPt = NF->begin()->begin(); // Just add all the struct element types. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); Value *TheAlloca = new AllocaInst(AgTy, nullptr, "", InsertPt); StructType *STy = cast<StructType>(AgTy); Value *Idxs[2] = { ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr }; for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i); Value *Idx = GetElementPtrInst::Create(TheAlloca, Idxs, TheAlloca->getName()+"."+Twine(i), InsertPt); I2->setName(I->getName()+"."+Twine(i)); new StoreInst(I2++, Idx, InsertPt); } // Anything that used the arg should now use the alloca. I->replaceAllUsesWith(TheAlloca); TheAlloca->takeName(I); AA.replaceWithNewValue(I, TheAlloca); // If the alloca is used in a call, we must clear the tail flag since // the callee now uses an alloca from the caller. for (User *U : TheAlloca->users()) { CallInst *Call = dyn_cast<CallInst>(U); if (!Call) continue; Call->setTailCall(false); } continue; } if (I->use_empty()) { AA.deleteValue(I); continue; } // Otherwise, if we promoted this argument, then all users are load // instructions (or GEPs with only load users), and all loads should be // using the new argument that we added. ScalarizeTable &ArgIndices = ScalarizedElements[I]; while (!I->use_empty()) { if (LoadInst *LI = dyn_cast<LoadInst>(I->user_back())) { assert(ArgIndices.begin()->empty() && "Load element should sort to front!"); I2->setName(I->getName()+".val"); LI->replaceAllUsesWith(I2); AA.replaceWithNewValue(LI, I2); LI->eraseFromParent(); DEBUG(dbgs() << "*** Promoted load of argument '" << I->getName() << "' in function '" << F->getName() << "'\n"); } else { GetElementPtrInst *GEP = cast<GetElementPtrInst>(I->user_back()); IndicesVector Operands; Operands.reserve(GEP->getNumIndices()); for (User::op_iterator II = GEP->idx_begin(), IE = GEP->idx_end(); II != IE; ++II) Operands.push_back(cast<ConstantInt>(*II)->getSExtValue()); // GEPs with a single 0 index can be merged with direct loads if (Operands.size() == 1 && Operands.front() == 0) Operands.clear(); Function::arg_iterator TheArg = I2; for (ScalarizeTable::iterator It = ArgIndices.begin(); *It != Operands; ++It, ++TheArg) { assert(It != ArgIndices.end() && "GEP not handled??"); } std::string NewName = I->getName(); for (unsigned i = 0, e = Operands.size(); i != e; ++i) { NewName += "." + utostr(Operands[i]); } NewName += ".val"; TheArg->setName(NewName); DEBUG(dbgs() << "*** Promoted agg argument '" << TheArg->getName() << "' of function '" << NF->getName() << "'\n"); // All of the uses must be load instructions. Replace them all with // the argument specified by ArgNo. while (!GEP->use_empty()) { LoadInst *L = cast<LoadInst>(GEP->user_back()); L->replaceAllUsesWith(TheArg); AA.replaceWithNewValue(L, TheArg); L->eraseFromParent(); } AA.deleteValue(GEP); GEP->eraseFromParent(); } } // Increment I2 past all of the arguments added for this promoted pointer. std::advance(I2, ArgIndices.size()); } // Tell the alias analysis that the old function is about to disappear. AA.replaceWithNewValue(F, NF); NF_CGN->stealCalledFunctionsFrom(CG[F]); // Now that the old function is dead, delete it. If there is a dangling // reference to the CallgraphNode, just leave the dead function around for // someone else to nuke. CallGraphNode *CGN = CG[F]; if (CGN->getNumReferences() == 0) delete CG.removeFunctionFromModule(CGN); else F->setLinkage(Function::ExternalLinkage); return NF_CGN; }
// Check to see if this function returns one or more constants. If so, replace // all callers that use those return values with the constant value. This will // leave in the actual return values and instructions, but deadargelim will // clean that up. // // Additionally if a function always returns one of its arguments directly, // callers will be updated to use the value they pass in directly instead of // using the return value. bool IPCP::PropagateConstantReturn(Function &F) { if (F.getReturnType()->isVoidTy()) return false; // No return value. // If this function could be overridden later in the link stage, we can't // propagate information about its results into callers. if (F.mayBeOverridden()) return false; // Check to see if this function returns a constant. SmallVector<Value *,4> RetVals; StructType *STy = dyn_cast<StructType>(F.getReturnType()); if (STy) for (unsigned i = 0, e = STy->getNumElements(); i < e; ++i) RetVals.push_back(UndefValue::get(STy->getElementType(i))); else RetVals.push_back(UndefValue::get(F.getReturnType())); unsigned NumNonConstant = 0; for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) { for (unsigned i = 0, e = RetVals.size(); i != e; ++i) { // Already found conflicting return values? Value *RV = RetVals[i]; if (!RV) continue; // Find the returned value Value *V; if (!STy) V = RI->getOperand(0); else V = FindInsertedValue(RI->getOperand(0), i); if (V) { // Ignore undefs, we can change them into anything if (isa<UndefValue>(V)) continue; // Try to see if all the rets return the same constant or argument. if (isa<Constant>(V) || isa<Argument>(V)) { if (isa<UndefValue>(RV)) { // No value found yet? Try the current one. RetVals[i] = V; continue; } // Returning the same value? Good. if (RV == V) continue; } } // Different or no known return value? Don't propagate this return // value. RetVals[i] = 0; // All values non constant? Stop looking. if (++NumNonConstant == RetVals.size()) return false; } } // If we got here, the function returns at least one constant value. Loop // over all users, replacing any uses of the return value with the returned // constant. bool MadeChange = false; for (Value::use_iterator UI = F.use_begin(), E = F.use_end(); UI != E; ++UI) { CallSite CS(*UI); Instruction* Call = CS.getInstruction(); // Not a call instruction or a call instruction that's not calling F // directly? if (!Call || !CS.isCallee(UI)) continue; // Call result not used? if (Call->use_empty()) continue; MadeChange = true; if (STy == 0) { Value* New = RetVals[0]; if (Argument *A = dyn_cast<Argument>(New)) // Was an argument returned? Then find the corresponding argument in // the call instruction and use that. New = CS.getArgument(A->getArgNo()); Call->replaceAllUsesWith(New); continue; } for (Value::use_iterator I = Call->use_begin(), E = Call->use_end(); I != E;) { Instruction *Ins = cast<Instruction>(*I); // Increment now, so we can remove the use ++I; // Find the index of the retval to replace with int index = -1; if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Ins)) if (EV->hasIndices()) index = *EV->idx_begin(); // If this use uses a specific return value, and we have a replacement, // replace it. if (index != -1) { Value *New = RetVals[index]; if (New) { if (Argument *A = dyn_cast<Argument>(New)) // Was an argument returned? Then find the corresponding argument in // the call instruction and use that. New = CS.getArgument(A->getArgNo()); Ins->replaceAllUsesWith(New); Ins->eraseFromParent(); } } } } if (MadeChange) ++NumReturnValProped; return MadeChange; }
/// cmpType - compares two types, /// defines total ordering among the types set. /// See method declaration comments for more details. int FunctionComparator::cmpTypes(Type *TyL, Type *TyR) const { PointerType *PTyL = dyn_cast<PointerType>(TyL); PointerType *PTyR = dyn_cast<PointerType>(TyR); const DataLayout &DL = FnL->getParent()->getDataLayout(); if (PTyL && PTyL->getAddressSpace() == 0) TyL = DL.getIntPtrType(TyL); if (PTyR && PTyR->getAddressSpace() == 0) TyR = DL.getIntPtrType(TyR); if (TyL == TyR) return 0; if (int Res = cmpNumbers(TyL->getTypeID(), TyR->getTypeID())) return Res; switch (TyL->getTypeID()) { default: llvm_unreachable("Unknown type!"); // Fall through in Release mode. LLVM_FALLTHROUGH; case Type::IntegerTyID: return cmpNumbers(cast<IntegerType>(TyL)->getBitWidth(), cast<IntegerType>(TyR)->getBitWidth()); // TyL == TyR would have returned true earlier, because types are uniqued. case Type::VoidTyID: case Type::FloatTyID: case Type::DoubleTyID: case Type::X86_FP80TyID: case Type::FP128TyID: case Type::PPC_FP128TyID: case Type::LabelTyID: case Type::MetadataTyID: case Type::TokenTyID: return 0; case Type::PointerTyID: { assert(PTyL && PTyR && "Both types must be pointers here."); return cmpNumbers(PTyL->getAddressSpace(), PTyR->getAddressSpace()); } case Type::StructTyID: { StructType *STyL = cast<StructType>(TyL); StructType *STyR = cast<StructType>(TyR); if (STyL->getNumElements() != STyR->getNumElements()) return cmpNumbers(STyL->getNumElements(), STyR->getNumElements()); if (STyL->isPacked() != STyR->isPacked()) return cmpNumbers(STyL->isPacked(), STyR->isPacked()); for (unsigned i = 0, e = STyL->getNumElements(); i != e; ++i) { if (int Res = cmpTypes(STyL->getElementType(i), STyR->getElementType(i))) return Res; } return 0; } case Type::FunctionTyID: { FunctionType *FTyL = cast<FunctionType>(TyL); FunctionType *FTyR = cast<FunctionType>(TyR); if (FTyL->getNumParams() != FTyR->getNumParams()) return cmpNumbers(FTyL->getNumParams(), FTyR->getNumParams()); if (FTyL->isVarArg() != FTyR->isVarArg()) return cmpNumbers(FTyL->isVarArg(), FTyR->isVarArg()); if (int Res = cmpTypes(FTyL->getReturnType(), FTyR->getReturnType())) return Res; for (unsigned i = 0, e = FTyL->getNumParams(); i != e; ++i) { if (int Res = cmpTypes(FTyL->getParamType(i), FTyR->getParamType(i))) return Res; } return 0; } case Type::ArrayTyID: case Type::VectorTyID: { auto *STyL = cast<SequentialType>(TyL); auto *STyR = cast<SequentialType>(TyR); if (STyL->getNumElements() != STyR->getNumElements()) return cmpNumbers(STyL->getNumElements(), STyR->getNumElements()); return cmpTypes(STyL->getElementType(), STyR->getElementType()); } } }
/// cmpType - compares two types, /// defines total ordering among the types set. /// See method declaration comments for more details. int FunctionComparator::cmpType(Type *TyL, Type *TyR) const { PointerType *PTyL = dyn_cast<PointerType>(TyL); PointerType *PTyR = dyn_cast<PointerType>(TyR); if (DL) { if (PTyL && PTyL->getAddressSpace() == 0) TyL = DL->getIntPtrType(TyL); if (PTyR && PTyR->getAddressSpace() == 0) TyR = DL->getIntPtrType(TyR); } if (TyL == TyR) return 0; if (int Res = cmpNumbers(TyL->getTypeID(), TyR->getTypeID())) return Res; switch (TyL->getTypeID()) { default: llvm_unreachable("Unknown type!"); // Fall through in Release mode. case Type::IntegerTyID: case Type::VectorTyID: // TyL == TyR would have returned true earlier. return cmpNumbers((uint64_t)TyL, (uint64_t)TyR); case Type::VoidTyID: case Type::FloatTyID: case Type::DoubleTyID: case Type::X86_FP80TyID: case Type::FP128TyID: case Type::PPC_FP128TyID: case Type::LabelTyID: case Type::MetadataTyID: return 0; case Type::PointerTyID: { assert(PTyL && PTyR && "Both types must be pointers here."); return cmpNumbers(PTyL->getAddressSpace(), PTyR->getAddressSpace()); } case Type::StructTyID: { StructType *STyL = cast<StructType>(TyL); StructType *STyR = cast<StructType>(TyR); if (STyL->getNumElements() != STyR->getNumElements()) return cmpNumbers(STyL->getNumElements(), STyR->getNumElements()); if (STyL->isPacked() != STyR->isPacked()) return cmpNumbers(STyL->isPacked(), STyR->isPacked()); for (unsigned i = 0, e = STyL->getNumElements(); i != e; ++i) { if (int Res = cmpType(STyL->getElementType(i), STyR->getElementType(i))) return Res; } return 0; } case Type::FunctionTyID: { FunctionType *FTyL = cast<FunctionType>(TyL); FunctionType *FTyR = cast<FunctionType>(TyR); if (FTyL->getNumParams() != FTyR->getNumParams()) return cmpNumbers(FTyL->getNumParams(), FTyR->getNumParams()); if (FTyL->isVarArg() != FTyR->isVarArg()) return cmpNumbers(FTyL->isVarArg(), FTyR->isVarArg()); if (int Res = cmpType(FTyL->getReturnType(), FTyR->getReturnType())) return Res; for (unsigned i = 0, e = FTyL->getNumParams(); i != e; ++i) { if (int Res = cmpType(FTyL->getParamType(i), FTyR->getParamType(i))) return Res; } return 0; } case Type::ArrayTyID: { ArrayType *ATyL = cast<ArrayType>(TyL); ArrayType *ATyR = cast<ArrayType>(TyR); if (ATyL->getNumElements() != ATyR->getNumElements()) return cmpNumbers(ATyL->getNumElements(), ATyR->getNumElements()); return cmpType(ATyL->getElementType(), ATyR->getElementType()); } } }
/// DoPromotion - This method actually performs the promotion of the specified /// arguments, and returns the new function. At this point, we know that it's /// safe to do so. static Function * doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote, SmallPtrSetImpl<Argument *> &ByValArgsToTransform, Optional<function_ref<void(CallSite OldCS, CallSite NewCS)>> ReplaceCallSite) { // Start by computing a new prototype for the function, which is the same as // the old function, but has modified arguments. FunctionType *FTy = F->getFunctionType(); std::vector<Type *> Params; using ScalarizeTable = std::set<std::pair<Type *, IndicesVector>>; // ScalarizedElements - If we are promoting a pointer that has elements // accessed out of it, keep track of which elements are accessed so that we // can add one argument for each. // // Arguments that are directly loaded will have a zero element value here, to // handle cases where there are both a direct load and GEP accesses. std::map<Argument *, ScalarizeTable> ScalarizedElements; // OriginalLoads - Keep track of a representative load instruction from the // original function so that we can tell the alias analysis implementation // what the new GEP/Load instructions we are inserting look like. // We need to keep the original loads for each argument and the elements // of the argument that are accessed. std::map<std::pair<Argument *, IndicesVector>, LoadInst *> OriginalLoads; // Attribute - Keep track of the parameter attributes for the arguments // that we are *not* promoting. For the ones that we do promote, the parameter // attributes are lost SmallVector<AttributeSet, 8> ArgAttrVec; AttributeList PAL = F->getAttributes(); // First, determine the new argument list unsigned ArgNo = 0; for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I, ++ArgNo) { if (ByValArgsToTransform.count(&*I)) { // Simple byval argument? Just add all the struct element types. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); StructType *STy = cast<StructType>(AgTy); Params.insert(Params.end(), STy->element_begin(), STy->element_end()); ArgAttrVec.insert(ArgAttrVec.end(), STy->getNumElements(), AttributeSet()); ++NumByValArgsPromoted; } else if (!ArgsToPromote.count(&*I)) { // Unchanged argument Params.push_back(I->getType()); ArgAttrVec.push_back(PAL.getParamAttributes(ArgNo)); } else if (I->use_empty()) { // Dead argument (which are always marked as promotable) ++NumArgumentsDead; // There may be remaining metadata uses of the argument for things like // llvm.dbg.value. Replace them with undef. I->replaceAllUsesWith(UndefValue::get(I->getType())); } else { // Okay, this is being promoted. This means that the only uses are loads // or GEPs which are only used by loads // In this table, we will track which indices are loaded from the argument // (where direct loads are tracked as no indices). ScalarizeTable &ArgIndices = ScalarizedElements[&*I]; for (User *U : I->users()) { Instruction *UI = cast<Instruction>(U); Type *SrcTy; if (LoadInst *L = dyn_cast<LoadInst>(UI)) SrcTy = L->getType(); else SrcTy = cast<GetElementPtrInst>(UI)->getSourceElementType(); IndicesVector Indices; Indices.reserve(UI->getNumOperands() - 1); // Since loads will only have a single operand, and GEPs only a single // non-index operand, this will record direct loads without any indices, // and gep+loads with the GEP indices. for (User::op_iterator II = UI->op_begin() + 1, IE = UI->op_end(); II != IE; ++II) Indices.push_back(cast<ConstantInt>(*II)->getSExtValue()); // GEPs with a single 0 index can be merged with direct loads if (Indices.size() == 1 && Indices.front() == 0) Indices.clear(); ArgIndices.insert(std::make_pair(SrcTy, Indices)); LoadInst *OrigLoad; if (LoadInst *L = dyn_cast<LoadInst>(UI)) OrigLoad = L; else // Take any load, we will use it only to update Alias Analysis OrigLoad = cast<LoadInst>(UI->user_back()); OriginalLoads[std::make_pair(&*I, Indices)] = OrigLoad; } // Add a parameter to the function for each element passed in. for (const auto &ArgIndex : ArgIndices) { // not allowed to dereference ->begin() if size() is 0 Params.push_back(GetElementPtrInst::getIndexedType( cast<PointerType>(I->getType()->getScalarType())->getElementType(), ArgIndex.second)); ArgAttrVec.push_back(AttributeSet()); assert(Params.back()); } if (ArgIndices.size() == 1 && ArgIndices.begin()->second.empty()) ++NumArgumentsPromoted; else ++NumAggregatesPromoted; } } Type *RetTy = FTy->getReturnType(); // Construct the new function type using the new arguments. FunctionType *NFTy = FunctionType::get(RetTy, Params, FTy->isVarArg()); // Create the new function body and insert it into the module. Function *NF = Function::Create(NFTy, F->getLinkage(), F->getName()); NF->copyAttributesFrom(F); // Patch the pointer to LLVM function in debug info descriptor. NF->setSubprogram(F->getSubprogram()); F->setSubprogram(nullptr); DEBUG(dbgs() << "ARG PROMOTION: Promoting to:" << *NF << "\n" << "From: " << *F); // Recompute the parameter attributes list based on the new arguments for // the function. NF->setAttributes(AttributeList::get(F->getContext(), PAL.getFnAttributes(), PAL.getRetAttributes(), ArgAttrVec)); ArgAttrVec.clear(); F->getParent()->getFunctionList().insert(F->getIterator(), NF); NF->takeName(F); // Loop over all of the callers of the function, transforming the call sites // to pass in the loaded pointers. // SmallVector<Value *, 16> Args; while (!F->use_empty()) { CallSite CS(F->user_back()); assert(CS.getCalledFunction() == F); Instruction *Call = CS.getInstruction(); const AttributeList &CallPAL = CS.getAttributes(); // Loop over the operands, inserting GEP and loads in the caller as // appropriate. CallSite::arg_iterator AI = CS.arg_begin(); ArgNo = 0; for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I, ++AI, ++ArgNo) if (!ArgsToPromote.count(&*I) && !ByValArgsToTransform.count(&*I)) { Args.push_back(*AI); // Unmodified argument ArgAttrVec.push_back(CallPAL.getParamAttributes(ArgNo)); } else if (ByValArgsToTransform.count(&*I)) { // Emit a GEP and load for each element of the struct. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); StructType *STy = cast<StructType>(AgTy); Value *Idxs[2] = { ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr}; for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i); Value *Idx = GetElementPtrInst::Create( STy, *AI, Idxs, (*AI)->getName() + "." + Twine(i), Call); // TODO: Tell AA about the new values? Args.push_back(new LoadInst(Idx, Idx->getName() + ".val", Call)); ArgAttrVec.push_back(AttributeSet()); } } else if (!I->use_empty()) { // Non-dead argument: insert GEPs and loads as appropriate. ScalarizeTable &ArgIndices = ScalarizedElements[&*I]; // Store the Value* version of the indices in here, but declare it now // for reuse. std::vector<Value *> Ops; for (const auto &ArgIndex : ArgIndices) { Value *V = *AI; LoadInst *OrigLoad = OriginalLoads[std::make_pair(&*I, ArgIndex.second)]; if (!ArgIndex.second.empty()) { Ops.reserve(ArgIndex.second.size()); Type *ElTy = V->getType(); for (auto II : ArgIndex.second) { // Use i32 to index structs, and i64 for others (pointers/arrays). // This satisfies GEP constraints. Type *IdxTy = (ElTy->isStructTy() ? Type::getInt32Ty(F->getContext()) : Type::getInt64Ty(F->getContext())); Ops.push_back(ConstantInt::get(IdxTy, II)); // Keep track of the type we're currently indexing. if (auto *ElPTy = dyn_cast<PointerType>(ElTy)) ElTy = ElPTy->getElementType(); else ElTy = cast<CompositeType>(ElTy)->getTypeAtIndex(II); } // And create a GEP to extract those indices. V = GetElementPtrInst::Create(ArgIndex.first, V, Ops, V->getName() + ".idx", Call); Ops.clear(); } // Since we're replacing a load make sure we take the alignment // of the previous load. LoadInst *newLoad = new LoadInst(V, V->getName() + ".val", Call); newLoad->setAlignment(OrigLoad->getAlignment()); // Transfer the AA info too. AAMDNodes AAInfo; OrigLoad->getAAMetadata(AAInfo); newLoad->setAAMetadata(AAInfo); Args.push_back(newLoad); ArgAttrVec.push_back(AttributeSet()); } } // Push any varargs arguments on the list. for (; AI != CS.arg_end(); ++AI, ++ArgNo) { Args.push_back(*AI); ArgAttrVec.push_back(CallPAL.getParamAttributes(ArgNo)); } SmallVector<OperandBundleDef, 1> OpBundles; CS.getOperandBundlesAsDefs(OpBundles); CallSite NewCS; if (InvokeInst *II = dyn_cast<InvokeInst>(Call)) { NewCS = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(), Args, OpBundles, "", Call); } else { auto *NewCall = CallInst::Create(NF, Args, OpBundles, "", Call); NewCall->setTailCallKind(cast<CallInst>(Call)->getTailCallKind()); NewCS = NewCall; } NewCS.setCallingConv(CS.getCallingConv()); NewCS.setAttributes( AttributeList::get(F->getContext(), CallPAL.getFnAttributes(), CallPAL.getRetAttributes(), ArgAttrVec)); NewCS->setDebugLoc(Call->getDebugLoc()); uint64_t W; if (Call->extractProfTotalWeight(W)) NewCS->setProfWeight(W); Args.clear(); ArgAttrVec.clear(); // Update the callgraph to know that the callsite has been transformed. if (ReplaceCallSite) (*ReplaceCallSite)(CS, NewCS); if (!Call->use_empty()) { Call->replaceAllUsesWith(NewCS.getInstruction()); NewCS->takeName(Call); } // Finally, remove the old call from the program, reducing the use-count of // F. Call->eraseFromParent(); } const DataLayout &DL = F->getParent()->getDataLayout(); // Since we have now created the new function, splice the body of the old // function right into the new function, leaving the old rotting hulk of the // function empty. NF->getBasicBlockList().splice(NF->begin(), F->getBasicBlockList()); // Loop over the argument list, transferring uses of the old arguments over to // the new arguments, also transferring over the names as well. for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(), I2 = NF->arg_begin(); I != E; ++I) { if (!ArgsToPromote.count(&*I) && !ByValArgsToTransform.count(&*I)) { // If this is an unmodified argument, move the name and users over to the // new version. I->replaceAllUsesWith(&*I2); I2->takeName(&*I); ++I2; continue; } if (ByValArgsToTransform.count(&*I)) { // In the callee, we create an alloca, and store each of the new incoming // arguments into the alloca. Instruction *InsertPt = &NF->begin()->front(); // Just add all the struct element types. Type *AgTy = cast<PointerType>(I->getType())->getElementType(); Value *TheAlloca = new AllocaInst(AgTy, DL.getAllocaAddrSpace(), nullptr, I->getParamAlignment(), "", InsertPt); StructType *STy = cast<StructType>(AgTy); Value *Idxs[2] = {ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr}; for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), i); Value *Idx = GetElementPtrInst::Create( AgTy, TheAlloca, Idxs, TheAlloca->getName() + "." + Twine(i), InsertPt); I2->setName(I->getName() + "." + Twine(i)); new StoreInst(&*I2++, Idx, InsertPt); } // Anything that used the arg should now use the alloca. I->replaceAllUsesWith(TheAlloca); TheAlloca->takeName(&*I); // If the alloca is used in a call, we must clear the tail flag since // the callee now uses an alloca from the caller. for (User *U : TheAlloca->users()) { CallInst *Call = dyn_cast<CallInst>(U); if (!Call) continue; Call->setTailCall(false); } continue; } if (I->use_empty()) continue; // Otherwise, if we promoted this argument, then all users are load // instructions (or GEPs with only load users), and all loads should be // using the new argument that we added. ScalarizeTable &ArgIndices = ScalarizedElements[&*I]; while (!I->use_empty()) { if (LoadInst *LI = dyn_cast<LoadInst>(I->user_back())) { assert(ArgIndices.begin()->second.empty() && "Load element should sort to front!"); I2->setName(I->getName() + ".val"); LI->replaceAllUsesWith(&*I2); LI->eraseFromParent(); DEBUG(dbgs() << "*** Promoted load of argument '" << I->getName() << "' in function '" << F->getName() << "'\n"); } else { GetElementPtrInst *GEP = cast<GetElementPtrInst>(I->user_back()); IndicesVector Operands; Operands.reserve(GEP->getNumIndices()); for (User::op_iterator II = GEP->idx_begin(), IE = GEP->idx_end(); II != IE; ++II) Operands.push_back(cast<ConstantInt>(*II)->getSExtValue()); // GEPs with a single 0 index can be merged with direct loads if (Operands.size() == 1 && Operands.front() == 0) Operands.clear(); Function::arg_iterator TheArg = I2; for (ScalarizeTable::iterator It = ArgIndices.begin(); It->second != Operands; ++It, ++TheArg) { assert(It != ArgIndices.end() && "GEP not handled??"); } std::string NewName = I->getName(); for (unsigned i = 0, e = Operands.size(); i != e; ++i) { NewName += "." + utostr(Operands[i]); } NewName += ".val"; TheArg->setName(NewName); DEBUG(dbgs() << "*** Promoted agg argument '" << TheArg->getName() << "' of function '" << NF->getName() << "'\n"); // All of the uses must be load instructions. Replace them all with // the argument specified by ArgNo. while (!GEP->use_empty()) { LoadInst *L = cast<LoadInst>(GEP->user_back()); L->replaceAllUsesWith(&*TheArg); L->eraseFromParent(); } GEP->eraseFromParent(); } } // Increment I2 past all of the arguments added for this promoted pointer. std::advance(I2, ArgIndices.size()); } return NF; }