bool Scalarizer::visitGetElementPtrInst(GetElementPtrInst &GEPI) { VectorType *VT = dyn_cast<VectorType>(GEPI.getType()); if (!VT) return false; IRBuilder<> Builder(&GEPI); unsigned NumElems = VT->getNumElements(); unsigned NumIndices = GEPI.getNumIndices(); Scatterer Base = scatter(&GEPI, GEPI.getOperand(0)); SmallVector<Scatterer, 8> Ops; Ops.resize(NumIndices); for (unsigned I = 0; I < NumIndices; ++I) Ops[I] = scatter(&GEPI, GEPI.getOperand(I + 1)); ValueVector Res; Res.resize(NumElems); for (unsigned I = 0; I < NumElems; ++I) { SmallVector<Value *, 8> Indices; Indices.resize(NumIndices); for (unsigned J = 0; J < NumIndices; ++J) Indices[J] = Ops[J][I]; Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices, GEPI.getName() + ".i" + Twine(I)); if (GEPI.isInBounds()) if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I])) NewGEPI->setIsInBounds(); } gather(&GEPI, Res); return true; }
/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical /// register references and actual offsets. /// void PEI::replaceFrameIndices(MachineFunction &Fn) { if (!Fn.getFrameInfo()->hasStackObjects()) return; // Nothing to do? // Store SPAdj at exit of a basic block. SmallVector<int, 8> SPState; SPState.resize(Fn.getNumBlockIDs()); SmallPtrSet<MachineBasicBlock*, 8> Reachable; // Iterate over the reachable blocks in DFS order. for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> > DFI = df_ext_begin(&Fn, Reachable), DFE = df_ext_end(&Fn, Reachable); DFI != DFE; ++DFI) { int SPAdj = 0; // Check the exit state of the DFS stack predecessor. if (DFI.getPathLength() >= 2) { MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); assert(Reachable.count(StackPred) && "DFS stack predecessor is already visited.\n"); SPAdj = SPState[StackPred->getNumber()]; } MachineBasicBlock *BB = *DFI; replaceFrameIndices(BB, Fn, SPAdj); SPState[BB->getNumber()] = SPAdj; } // Handle the unreachable blocks. for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { if (Reachable.count(BB)) // Already handled in DFS traversal. continue; int SPAdj = 0; replaceFrameIndices(BB, Fn, SPAdj); } }
void RenameIndependentSubregs::distribute(const IntEqClasses &Classes, const SmallVectorImpl<SubRangeInfo> &SubRangeInfos, const SmallVectorImpl<LiveInterval*> &Intervals) const { unsigned NumClasses = Classes.getNumClasses(); SmallVector<unsigned, 8> VNIMapping; SmallVector<LiveInterval::SubRange*, 8> SubRanges; BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); for (const SubRangeInfo &SRInfo : SubRangeInfos) { LiveInterval::SubRange &SR = *SRInfo.SR; unsigned NumValNos = SR.valnos.size(); VNIMapping.clear(); VNIMapping.reserve(NumValNos); SubRanges.clear(); SubRanges.resize(NumClasses-1, nullptr); for (unsigned I = 0; I < NumValNos; ++I) { const VNInfo &VNI = *SR.valnos[I]; unsigned LocalID = SRInfo.ConEQ.getEqClass(&VNI); unsigned ID = Classes[LocalID + SRInfo.Index]; VNIMapping.push_back(ID); if (ID > 0 && SubRanges[ID-1] == nullptr) SubRanges[ID-1] = Intervals[ID]->createSubRange(Allocator, SR.LaneMask); } DistributeRange(SR, SubRanges.data(), VNIMapping); } }
// binary zip operation, e.g. Plus // If allowBroadcast then one can be a sub-dimension of the other (if layout then only for rows, otherwise for cols, too). // This also helpfully resizes the children if not yet sized. void ComputationNodeBase::ValidateBinaryZip(bool isFinalValidationPass, bool allowBroadcast) { assert(m_inputs.size() == 2); ComputationNodeBase::Validate(isFinalValidationPass); InferMBLayoutFromInputsForStandardCase(isFinalValidationPass); ValidateInferBinaryInputDims(); if (isFinalValidationPass) ValidateMBLayout(Input(0), Input(1)); // result has tensor shape with dimensions being the max over both let shape0 = GetInputSampleLayout(0); let shape1 = GetInputSampleLayout(1); SmallVector<size_t> dims = shape0.GetDims(); if (shape1.GetRank() > dims.size()) dims.resize(shape1.GetRank(), 1); // pad with ones // If rank of [0] is higher than we only need to take max over rank [1]. // If rank of [1] is higher then we have padded to equal lentgh. for (size_t k = 0; k < shape1.GetRank(); k++) { size_t dim1 = shape1[k]; // BUGBUG: We must consider the allowBroadcast flag here. if (dims[k] <= 1 && dim1 != 0) // is [0] broadcasting (1) or unspecified (0)? dims[k] = dim1; // then use dimension we broadcast to else if (dim1 <= 1 && dims[k] != 0) // if [1] is broadcasting or unspecified ; // then dims is already correct else if (isFinalValidationPass && dim1 != dims[k]) // no broadcasting or unspecified: they must match InvalidArgument("%ls: Input dimensions [%s] and [%s] are not compatible.", NodeDescription().c_str(), string(shape0).c_str(), string(shape1).c_str()); } SetDims(TensorShape(dims), HasMBLayout()); }
// Topologically sorts the basic blocks in the function and writes the ordering // into the supplied unique vector. The back and incoming edges must have been // computed first. void LiveIRVariables::computeTopologicalOrdering(Function &F, UniqueVector<BasicBlock *> &Ordering) { assert(IncomingEdges.size() == F.size() && "Incoming edges not computed yet!"); SmallVector<unsigned, 256> ProcessedIncomingEdges; ProcessedIncomingEdges.resize(F.size(), 0); SmallVector<BasicBlock *, 256> WorkList; WorkList.push_back(&F.getEntryBlock()); while (!WorkList.empty()) { BasicBlock *BB = WorkList.back(); WorkList.pop_back(); DEBUG(dbgs() << "Assigning topological order " << Ordering.size()); DEBUG(dbgs() << " to basic block with DFS order "); DEBUG(dbgs() << (DFSOrdering.idFor(BB) - 1) << "\n"); Ordering.insert(BB); for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB); SI != SE; ++SI) { if (BackEdges.count(std::make_pair(BB, *SI))) continue; unsigned DFSID = DFSOrdering.idFor(*SI) - 1; unsigned ProcessedEdges = ++ProcessedIncomingEdges[DFSID]; if (ProcessedEdges == IncomingEdges[DFSID]) WorkList.push_back(*SI); } } }
std::string DebugIR::getPath() { SmallVector<char, 16> Path; sys::path::append(Path, Directory, Filename); Path.resize(Filename.size() + Directory.size() + 2); Path[Filename.size() + Directory.size() + 1] = '\0'; return std::string(Path.data()); }
/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical /// register references and actual offsets. void PEI::replaceFrameIndices(MachineFunction &MF) { const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); if (!TFI.needsFrameIndexResolution(MF)) return; // Store SPAdj at exit of a basic block. SmallVector<int, 8> SPState; SPState.resize(MF.getNumBlockIDs()); df_iterator_default_set<MachineBasicBlock*> Reachable; // Iterate over the reachable blocks in DFS order. for (auto DFI = df_ext_begin(&MF, Reachable), DFE = df_ext_end(&MF, Reachable); DFI != DFE; ++DFI) { int SPAdj = 0; // Check the exit state of the DFS stack predecessor. if (DFI.getPathLength() >= 2) { MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); assert(Reachable.count(StackPred) && "DFS stack predecessor is already visited.\n"); SPAdj = SPState[StackPred->getNumber()]; } MachineBasicBlock *BB = *DFI; replaceFrameIndices(BB, MF, SPAdj); SPState[BB->getNumber()] = SPAdj; } // Handle the unreachable blocks. for (auto &BB : MF) { if (Reachable.count(&BB)) // Already handled in DFS traversal. continue; int SPAdj = 0; replaceFrameIndices(&BB, MF, SPAdj); } }
llvm::Constant* CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral* E) { //assert(!E->getType()->isPointerType() && "Strings are always arrays"); // TEMP only handle 1 byte per char SmallString<64> Str(E->getValue()); Str.resize(E->getByteLength()); //return llvm::ConstantDataArray::getString(context, Str, false); return llvm::ConstantDataArray::getString(context, Str, true); // add 0 #if 0 // Don't emit it as the address of the string, emit the string data itself // as an inline array. if (E->getCharByteWidth() == 1) { SmallString<64> Str(E->getString()); // Resize the string to the right size, which is indicated by its type. const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType()); Str.resize(CAT->getSize().getZExtValue()); return llvm::ConstantDataArray::getString(VMContext, Str, false); } llvm::ArrayType *AType = cast<llvm::ArrayType>(getTypes().ConvertType(E->getType())); llvm::Type *ElemTy = AType->getElementType(); unsigned NumElements = AType->getNumElements(); // Wide strings have either 2-byte or 4-byte elements. if (ElemTy->getPrimitiveSizeInBits() == 16) { SmallVector<uint16_t, 32> Elements; Elements.reserve(NumElements); for(unsigned i = 0, e = E->getLength(); i != e; ++i) Elements.push_back(E->getCodeUnit(i)); Elements.resize(NumElements); return llvm::ConstantDataArray::get(VMContext, Elements); } assert(ElemTy->getPrimitiveSizeInBits() == 32); SmallVector<uint32_t, 32> Elements; Elements.reserve(NumElements); for(unsigned i = 0, e = E->getLength(); i != e; ++i) Elements.push_back(E->getCodeUnit(i)); Elements.resize(NumElements); return llvm::ConstantDataArray::get(VMContext, Elements); #endif }
/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical /// register references and actual offsets. /// void PEI::replaceFrameIndices(MachineFunction &Fn) { const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering(); if (!TFI.needsFrameIndexResolution(Fn)) return; MachineModuleInfo &MMI = Fn.getMMI(); const Function *F = Fn.getFunction(); const Function *ParentF = MMI.getWinEHParent(F); unsigned FrameReg; if (F == ParentF) { WinEHFuncInfo &FuncInfo = MMI.getWinEHFuncInfo(Fn.getFunction()); // FIXME: This should be unconditional but we have bugs in the preparation // pass. if (FuncInfo.UnwindHelpFrameIdx != INT_MAX) FuncInfo.UnwindHelpFrameOffset = TFI.getFrameIndexReferenceFromSP( Fn, FuncInfo.UnwindHelpFrameIdx, FrameReg); for (WinEHTryBlockMapEntry &TBME : FuncInfo.TryBlockMap) { for (WinEHHandlerType &H : TBME.HandlerArray) { unsigned UnusedReg; if (H.CatchObj.FrameIndex == INT_MAX) H.CatchObj.FrameOffset = INT_MAX; else H.CatchObj.FrameOffset = TFI.getFrameIndexReference(Fn, H.CatchObj.FrameIndex, UnusedReg); } } } // Store SPAdj at exit of a basic block. SmallVector<int, 8> SPState; SPState.resize(Fn.getNumBlockIDs()); SmallPtrSet<MachineBasicBlock*, 8> Reachable; // Iterate over the reachable blocks in DFS order. for (auto DFI = df_ext_begin(&Fn, Reachable), DFE = df_ext_end(&Fn, Reachable); DFI != DFE; ++DFI) { int SPAdj = 0; // Check the exit state of the DFS stack predecessor. if (DFI.getPathLength() >= 2) { MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); assert(Reachable.count(StackPred) && "DFS stack predecessor is already visited.\n"); SPAdj = SPState[StackPred->getNumber()]; } MachineBasicBlock *BB = *DFI; replaceFrameIndices(BB, Fn, SPAdj); SPState[BB->getNumber()] = SPAdj; } // Handle the unreachable blocks. for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { if (Reachable.count(BB)) // Already handled in DFS traversal. continue; int SPAdj = 0; replaceFrameIndices(BB, Fn, SPAdj); } }
SmallVector<uint8_t, 64> GetShadowBytes(const SmallVectorImpl<ASanStackVariableDescription> &Vars, const ASanStackFrameLayout &Layout) { assert(Vars.size() > 0); SmallVector<uint8_t, 64> SB; SB.clear(); const size_t Granularity = Layout.Granularity; SB.resize(Vars[0].Offset / Granularity, kAsanStackLeftRedzoneMagic); for (const auto &Var : Vars) { SB.resize(Var.Offset / Granularity, kAsanStackMidRedzoneMagic); SB.resize(SB.size() + Var.Size / Granularity, 0); if (Var.Size % Granularity) SB.push_back(Var.Size % Granularity); } SB.resize(Layout.FrameSize / Granularity, kAsanStackRightRedzoneMagic); return SB; }
Constant *ShadowStackGC::GetFrameMap(Function &F) { // doInitialization creates the abstract type of this value. Type *VoidPtr = Type::getInt8PtrTy(F.getContext()); // Truncate the ShadowStackDescriptor if some metadata is null. unsigned NumMeta = 0; SmallVector<Constant*, 16> Metadata; for (unsigned I = 0; I != Roots.size(); ++I) { Constant *C = cast<Constant>(Roots[I].first->getArgOperand(1)); if (!C->isNullValue()) NumMeta = I + 1; Metadata.push_back(ConstantExpr::getBitCast(C, VoidPtr)); } Metadata.resize(NumMeta); Type *Int32Ty = Type::getInt32Ty(F.getContext()); Constant *BaseElts[] = { ConstantInt::get(Int32Ty, Roots.size(), false), ConstantInt::get(Int32Ty, NumMeta, false), }; Constant *DescriptorElts[] = { ConstantStruct::get(FrameMapTy, BaseElts), ConstantArray::get(ArrayType::get(VoidPtr, NumMeta), Metadata) }; Type *EltTys[] = { DescriptorElts[0]->getType(),DescriptorElts[1]->getType()}; StructType *STy = StructType::create(EltTys, "gc_map."+utostr(NumMeta)); Constant *FrameMap = ConstantStruct::get(STy, DescriptorElts); // FIXME: Is this actually dangerous as WritingAnLLVMPass.html claims? Seems // that, short of multithreaded LLVM, it should be safe; all that is // necessary is that a simple Module::iterator loop not be invalidated. // Appending to the GlobalVariable list is safe in that sense. // // All of the output passes emit globals last. The ExecutionEngine // explicitly supports adding globals to the module after // initialization. // // Still, if it isn't deemed acceptable, then this transformation needs // to be a ModulePass (which means it cannot be in the 'llc' pipeline // (which uses a FunctionPassManager (which segfaults (not asserts) if // provided a ModulePass))). Constant *GV = new GlobalVariable(*F.getParent(), FrameMap->getType(), true, GlobalVariable::InternalLinkage, FrameMap, "__gc_" + F.getName()); Constant *GEPIndices[2] = { ConstantInt::get(Type::getInt32Ty(F.getContext()), 0), ConstantInt::get(Type::getInt32Ty(F.getContext()), 0) }; return ConstantExpr::getGetElementPtr(GV, GEPIndices); }
/// promoteDestroyAddr - DestroyAddr is a composed operation merging /// load+strong_release. If the implicit load's value is available, explode it. /// /// Note that we handle the general case of a destroy_addr of a piece of the /// memory object, not just destroy_addrs of the entire thing. /// bool AllocOptimize::promoteDestroyAddr(DestroyAddrInst *DAI) { SILValue Address = DAI->getOperand(); // We cannot promote destroys of address-only types, because we can't expose // the load. SILType LoadTy = Address.getType().getObjectType(); if (LoadTy.isAddressOnly(Module)) return false; // If the box has escaped at this instruction, we can't safely promote the // load. if (hasEscapedAt(DAI)) return false; // Compute the access path down to the field so we can determine precise // def/use behavior. unsigned FirstElt = computeSubelement(Address, TheMemory); assert(FirstElt != ~0U && "destroy within enum projection is not valid"); unsigned NumLoadSubElements = getNumSubElements(LoadTy, Module); // Set up the bitvector of elements being demanded by the load. llvm::SmallBitVector RequiredElts(NumMemorySubElements); RequiredElts.set(FirstElt, FirstElt+NumLoadSubElements); SmallVector<std::pair<SILValue, unsigned>, 8> AvailableValues; AvailableValues.resize(NumMemorySubElements); // Find out if we have any available values. If no bits are demanded, we // trivially succeed. This can happen when there is a load of an empty struct. if (NumLoadSubElements != 0) { computeAvailableValues(DAI, RequiredElts, AvailableValues); // If some value is not available at this load point, then we fail. for (unsigned i = FirstElt, e = FirstElt+NumLoadSubElements; i != e; ++i) if (!AvailableValues[i].first.isValid()) return false; } // Aggregate together all of the subelements into something that has the same // type as the load did, and emit smaller) loads for any subelements that were // not available. auto NewVal = AggregateAvailableValues(DAI, LoadTy, Address, AvailableValues, FirstElt); ++NumDestroyAddrPromoted; DEBUG(llvm::dbgs() << " *** Promoting destroy_addr: " << *DAI << "\n"); DEBUG(llvm::dbgs() << " To value: " << *NewVal.getDef() << "\n"); SILBuilderWithScope(DAI).emitReleaseValueOperation(DAI->getLoc(), NewVal); DAI->eraseFromParent(); return true; }
void MatcherGen::EmitResultCode() { // Patterns that match nodes with (potentially multiple) chain inputs have to // merge them together into a token factor. This informs the generated code // what all the chained nodes are. if (!MatchedChainNodes.empty()) AddMatcher(new EmitMergeInputChainsMatcher (MatchedChainNodes.data(), MatchedChainNodes.size())); // Codegen the root of the result pattern, capturing the resulting values. SmallVector<unsigned, 8> Ops; EmitResultOperand(Pattern.getDstPattern(), Ops); // At this point, we have however many values the result pattern produces. // However, the input pattern might not need all of these. If there are // excess values at the end (such as implicit defs of condition codes etc) // just lop them off. This doesn't need to worry about glue or chains, just // explicit results. // unsigned NumSrcResults = Pattern.getSrcPattern()->getNumTypes(); // If the pattern also has (implicit) results, count them as well. if (!Pattern.getDstRegs().empty()) { // If the root came from an implicit def in the instruction handling stuff, // don't re-add it. Record *HandledReg = 0; const TreePatternNode *DstPat = Pattern.getDstPattern(); if (!DstPat->isLeaf() &&DstPat->getOperator()->isSubClassOf("Instruction")){ const CodeGenTarget &CGT = CGP.getTargetInfo(); CodeGenInstruction &II = CGT.getInstruction(DstPat->getOperator()); if (II.HasOneImplicitDefWithKnownVT(CGT) != MVT::Other) HandledReg = II.ImplicitDefs[0]; } for (unsigned i = 0; i != Pattern.getDstRegs().size(); ++i) { Record *Reg = Pattern.getDstRegs()[i]; if (!Reg->isSubClassOf("Register") || Reg == HandledReg) continue; ++NumSrcResults; } } assert(Ops.size() >= NumSrcResults && "Didn't provide enough results"); Ops.resize(NumSrcResults); // If the matched pattern covers nodes which define a glue result, emit a node // that tells the matcher about them so that it can update their results. if (!MatchedGlueResultNodes.empty()) AddMatcher(new MarkGlueResultsMatcher(MatchedGlueResultNodes.data(), MatchedGlueResultNodes.size())); AddMatcher(new CompleteMatchMatcher(Ops.data(), Ops.size(), Pattern)); }
/* * Renaming uses of V to uses of sigma * The rule of renaming is: * - All uses of V in the dominator tree of sigma(V) are renamed, except for the sigma itself, of course * - Uses of V in the dominance frontier of sigma(V) are renamed iff they are in PHI nodes (maybe this always happens) */ void vSSA::renameUsesToSigma(Value *V, PHINode *sigma) { DEBUG(dbgs() << "VSSA: Renaming uses of " << *V << " to " << *sigma << "\n"); BasicBlock *BB_next = sigma->getParent(); // Get the dominance frontier of the successor DominanceFrontier::iterator DF_BB = DF_->find(BB_next); // This vector of Instruction* points to the uses of V. // This auxiliary vector of pointers is used because the use_iterators are invalidated when we do the renaming SmallVector<Instruction*, 25> usepointers; unsigned i = 0, n = V->getNumUses(); usepointers.resize(n); for (Value::use_iterator uit = V->use_begin(), uend = V->use_end(); uit != uend; ++uit, ++i) usepointers[i] = dyn_cast<Instruction>(*uit); for (i = 0; i < n; ++i) { if (usepointers[i] == NULL) { continue; } if (usepointers[i] == sigma) { continue; } /* if (isa<GetElementPtrInst>(usepointers[i])) { continue; } */ BasicBlock *BB_user = usepointers[i]->getParent(); // Check if the use is in the dominator tree of sigma(V) if (DT_->dominates(BB_next, BB_user)){ usepointers[i]->replaceUsesOfWith(V, sigma); } // Check if the use is in the dominance frontier of sigma(V) else if ((DF_BB != DF_->end()) && (DF_BB->second.find(BB_user) != DF_BB->second.end())) { // Check if the user is a PHI node (it has to be, but only for precaution) if (PHINode *phi = dyn_cast<PHINode>(usepointers[i])) { for (unsigned i = 0, e = phi->getNumIncomingValues(); i < e; ++i) { Value *operand = phi->getIncomingValue(i); if (operand != V) continue; if (DT_->dominates(BB_next, phi->getIncomingBlock(i))) { phi->setIncomingValue(i, sigma); } } } } } }
StringRef LmbenchTiming::getName(EnumTy IG) { static SmallVector<std::string,NumGroups> InstGroupNames; if(InstGroupNames.size() == 0){ InstGroupNames.resize(NumGroups); auto& n = InstGroupNames; std::vector<std::pair<EnumTy,StringRef>> bits = { {Integer, "I"}, {I64, "I64"}, {Float,"F"}, {Double,"D"} }, ops = {{Add,"Add"},{Mul, "Mul"}, {Div, "Div"}, {Mod, "Mod"}}; for(auto bit : bits){ for(auto op : ops) n[bit.first|op.first] = (bit.second+op.second).str(); } } return InstGroupNames[IG]; }
void TypeMapTy::linkDefinedTypeBodies() { SmallVector<Type *, 16> Elements; for (StructType *SrcSTy : SrcDefinitionsToResolve) { StructType *DstSTy = cast<StructType>(MappedTypes[SrcSTy]); assert(DstSTy->isOpaque()); // Map the body of the source type over to a new body for the dest type. Elements.resize(SrcSTy->getNumElements()); for (unsigned I = 0, E = Elements.size(); I != E; ++I) Elements[I] = get(SrcSTy->getElementType(I)); DstSTy->setBody(Elements, SrcSTy->isPacked()); DstStructTypesSet.switchToNonOpaque(DstSTy); } SrcDefinitionsToResolve.clear(); DstResolvedOpaqueTypes.clear(); }
void StackColoring::calculateLiveIntervals() { for (auto IT : BlockLiveness) { BasicBlock *BB = IT.getFirst(); BlockLifetimeInfo &BlockInfo = IT.getSecond(); unsigned BBStart, BBEnd; std::tie(BBStart, BBEnd) = BlockInstRange[BB]; BitVector Started, Ended; Started.resize(NumAllocas); Ended.resize(NumAllocas); SmallVector<unsigned, 8> Start; Start.resize(NumAllocas); // LiveIn ranges start at the first instruction. for (unsigned AllocaNo = 0; AllocaNo < NumAllocas; ++AllocaNo) { if (BlockInfo.LiveIn.test(AllocaNo)) { Started.set(AllocaNo); Start[AllocaNo] = BBStart; } } for (auto &It : BBMarkers[BB]) { unsigned InstNo = It.first; bool IsStart = It.second.IsStart; unsigned AllocaNo = It.second.AllocaNo; if (IsStart) { assert(!Started.test(AllocaNo)); Started.set(AllocaNo); Ended.reset(AllocaNo); Start[AllocaNo] = InstNo; } else { assert(!Ended.test(AllocaNo)); if (Started.test(AllocaNo)) { LiveRanges[AllocaNo].AddRange(Start[AllocaNo], InstNo); Started.reset(AllocaNo); } Ended.set(AllocaNo); } } for (unsigned AllocaNo = 0; AllocaNo < NumAllocas; ++AllocaNo) if (Started.test(AllocaNo)) LiveRanges[AllocaNo].AddRange(Start[AllocaNo], BBEnd); } }
// Lower ".cpload $reg" to // "lui $gp, %hi(_gp_disp)" // "addiu $gp, $gp, %lo(_gp_disp)" // "addu $gp, $gp, $t9" void Cpu0MCInstLower::LowerCPLOAD(SmallVector<MCInst, 4>& MCInsts) { MCOperand GPReg = MCOperand::CreateReg(Cpu0::GP); MCOperand T9Reg = MCOperand::CreateReg(Cpu0::T9); StringRef SymName("_gp_disp"); const MCSymbol *Sym = Ctx->GetOrCreateSymbol(SymName); const MCSymbolRefExpr *MCSym; MCSym = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_Cpu0_ABS_HI, *Ctx); MCOperand SymHi = MCOperand::CreateExpr(MCSym); MCSym = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_Cpu0_ABS_LO, *Ctx); MCOperand SymLo = MCOperand::CreateExpr(MCSym); MCInsts.resize(3); CreateMCInst(MCInsts[0], Cpu0::LUi, GPReg, SymHi); CreateMCInst(MCInsts[1], Cpu0::ADDiu, GPReg, GPReg, SymLo); CreateMCInst(MCInsts[2], Cpu0::ADD, GPReg, GPReg, T9Reg); } // lbd document - mark - LowerCPLOAD
/// ReadProfilingBlock - Read the number of entries in the next profiling data /// packet and then accumulate the entries into 'Data'. static void ReadProfilingBlock(const char *ToolName, FILE *F, bool ShouldByteSwap, SmallVector<unsigned, 32> &Data) { // Read the number of entries... unsigned NumEntries = ReadProfilingNumEntries(ToolName, F, ShouldByteSwap); // Read in the data. SmallVector<unsigned, 8> TempSpace(NumEntries); ReadProfilingData<unsigned>(ToolName, F, TempSpace.data(), NumEntries); // Make sure we have enough space ... if (Data.size() < NumEntries) Data.resize(NumEntries, ProfileDataLoader::Uncounted); // Accumulate the data we just read into the existing data. for (unsigned i = 0; i < NumEntries; ++i) { unsigned Entry = ShouldByteSwap ? ByteSwap_32(TempSpace[i]) : TempSpace[i]; Data[i] = AddCounts(Entry, Data[i]); } }
// Lower ".cprestore offset" to "sw $gp, offset($sp)". void MipsMCInstLower::LowerCPRESTORE(int64_t Offset, SmallVector<MCInst, 4>& MCInsts) { assert(isInt<32>(Offset) && (Offset >= 0) && "Imm operand of .cprestore must be a non-negative 32-bit value."); MCOperand SPReg = MCOperand::CreateReg(Mips::SP), BaseReg = SPReg; MCOperand GPReg = MCOperand::CreateReg(Mips::GP); if (!isInt<16>(Offset)) { unsigned Hi = ((Offset + 0x8000) >> 16) & 0xffff; Offset &= 0xffff; MCOperand ATReg = MCOperand::CreateReg(Mips::AT); BaseReg = ATReg; // lui at,hi // addu at,at,sp MCInsts.resize(2); CreateMCInst(MCInsts[0], Mips::LUi, ATReg, MCOperand::CreateImm(Hi)); CreateMCInst(MCInsts[1], Mips::ADDu, ATReg, ATReg, SPReg); }
/// linkDefinedTypeBodies - Produce a body for an opaque type in the dest /// module from a type definition in the source module. void TypeMapTy::linkDefinedTypeBodies() { SmallVector<Type*, 16> Elements; SmallString<16> TmpName; // Note that processing entries in this loop (calling 'get') can add new // entries to the SrcDefinitionsToResolve vector. while (!SrcDefinitionsToResolve.empty()) { StructType *SrcSTy = SrcDefinitionsToResolve.pop_back_val(); StructType *DstSTy = cast<StructType>(MappedTypes[SrcSTy]); // TypeMap is a many-to-one mapping, if there were multiple types that // provide a body for DstSTy then previous iterations of this loop may have // already handled it. Just ignore this case. if (!DstSTy->isOpaque()) continue; assert(!SrcSTy->isOpaque() && "Not resolving a definition?"); // Map the body of the source type over to a new body for the dest type. Elements.resize(SrcSTy->getNumElements()); for (unsigned i = 0, e = Elements.size(); i != e; ++i) Elements[i] = getImpl(SrcSTy->getElementType(i)); DstSTy->setBody(Elements, SrcSTy->isPacked()); // If DstSTy has no name or has a longer name than STy, then viciously steal // STy's name. if (!SrcSTy->hasName()) continue; StringRef SrcName = SrcSTy->getName(); if (!DstSTy->hasName() || DstSTy->getName().size() > SrcName.size()) { TmpName.insert(TmpName.end(), SrcName.begin(), SrcName.end()); SrcSTy->setName(""); DstSTy->setName(TmpName.str()); TmpName.clear(); } } DstResolvedOpaqueTypes.clear(); }
// binary zip operation, e.g. Plus // If allowBroadcast then one can be a sub-dimension of the other (if layout then only for rows, otherwise for cols, too). // This also helpfully resizes the children if not yet sized. void ComputationNodeBase::ValidateBinaryZip(bool isFinalValidationPass, bool allowBroadcast) { assert(m_inputs.size() == 2); ComputationNodeBase::Validate(isFinalValidationPass); InferMBLayoutFromInputsForStandardCase(isFinalValidationPass); ValidateInferBinaryInputDims(); if (isFinalValidationPass && Input(0)->GetMBLayout() != Input(1)->GetMBLayout() && Input(0)->HasMBLayout() && Input(1)->HasMBLayout()) { LogicError("%ls: Minibatch layouts are not the same between arguments and might get out of sync during runtime. If this is by design, use ReconcileDynamicAxis() to forward layouts between nodes.", NodeDescription().c_str()); } // result has tensor shape with dimensions being the max over both let shape0 = GetInputSampleLayout(0); let shape1 = GetInputSampleLayout(1); SmallVector<size_t> dims = shape0.GetDims(); if (shape1.GetRank() > dims.size()) dims.resize(shape1.GetRank(), 1); // pad with ones // If rank of [0] is higher than we only need to take max over rank [1]. // If rank of [1] is higher then we have padded to equal lentgh. for (size_t k = 0; k < shape1.GetRank(); k++) { size_t dim1 = shape1[k]; // BUGBUG: We must consider the allowBroadcast flag here. if (dims[k] == 1) // is [0] broadcasting? dims[k] = dim1; // then use dimension we broadcast to else if (dim1 == 1) // if [1] is broadcasting ; // dims is already correct else if (isFinalValidationPass && dim1 != dims[k]) // no broadcasting: they must match InvalidArgument("%ls: Input dimensions [%s] and [%s] are not compatible.", NodeDescription().c_str(), string(shape0).c_str(), string(shape1).c_str()); } SetDims(TensorShape(dims), HasMBLayout()); }
bool Regex::match(StringRef String, SmallVectorImpl<StringRef> *Matches){ unsigned nmatch = Matches ? preg->re_nsub+1 : 0; // pmatch needs to have at least one element. SmallVector<llvm_regmatch_t, 8> pm; pm.resize(nmatch > 0 ? nmatch : 1); pm[0].rm_so = 0; pm[0].rm_eo = String.size(); int rc = llvm_regexec(preg, String.data(), nmatch, pm.data(), REG_STARTEND); if (rc == REG_NOMATCH) return false; if (rc != 0) { // regexec can fail due to invalid pattern or running out of memory. error = rc; return false; } // There was a match. if (Matches) { // match position requested Matches->clear(); for (unsigned i = 0; i != nmatch; ++i) { if (pm[i].rm_so == -1) { // this group didn't match Matches->push_back(StringRef()); continue; } assert(pm[i].rm_eo >= pm[i].rm_so); Matches->push_back(StringRef(String.data()+pm[i].rm_so, pm[i].rm_eo-pm[i].rm_so)); } } return true; }
// Lower ".cprestore offset" to "sw $gp, offset($sp)". void MipsMCInstLower::LowerCPRESTORE(const MachineInstr *MI, SmallVector<MCInst, 4>& MCInsts) { const MachineOperand &MO = MI->getOperand(0); assert(MO.isImm() && "CPRESTORE's operand must be an immediate."); unsigned Offset = MO.getImm(), Reg = Mips::SP; MCInst Sw; if (Offset >= 0x8000) { unsigned Hi = (Offset >> 16) + ((Offset & 0x8000) != 0); Offset &= 0xffff; Reg = Mips::AT; // lui at,hi // addu at,at,sp MCInsts.resize(2); MCInsts[0].setOpcode(Mips::LUi); MCInsts[0].addOperand(MCOperand::CreateReg(Mips::AT)); MCInsts[0].addOperand(MCOperand::CreateImm(Hi)); MCInsts[1].setOpcode(Mips::ADDu); MCInsts[1].addOperand(MCOperand::CreateReg(Mips::AT)); MCInsts[1].addOperand(MCOperand::CreateReg(Mips::AT)); MCInsts[1].addOperand(MCOperand::CreateReg(Mips::SP)); }
// Lower ".cprestore offset" to "st $gp, offset($sp)". void Cpu0MCInstLower::LowerCPRESTORE(int64_t Offset, SmallVector<MCInst, 4>& MCInsts) { assert(isInt<32>(Offset) && (Offset >= 0) && "Imm operand of .cprestore must be a non-negative 32-bit value."); MCOperand SPReg = MCOperand::CreateReg(Cpu0::SP), BaseReg = SPReg; MCOperand GPReg = MCOperand::CreateReg(Cpu0::GP); MCOperand ZEROReg = MCOperand::CreateReg(Cpu0::ZERO); if (!isInt<16>(Offset)) { unsigned Hi = ((Offset + 0x8000) >> 16) & 0xffff; Offset &= 0xffff; MCOperand ATReg = MCOperand::CreateReg(Cpu0::AT); BaseReg = ATReg; // addiu at,zero,hi // shl at,at,16 // add at,at,sp MCInsts.resize(3); CreateMCInst(MCInsts[0], Cpu0::ADDiu, ATReg, ZEROReg, MCOperand::CreateImm(Hi)); CreateMCInst(MCInsts[1], Cpu0::SHL, ATReg, ATReg, MCOperand::CreateImm(16)); CreateMCInst(MCInsts[2], Cpu0::ADD, ATReg, ATReg, SPReg); }
QualType ClassTemplateDecl::getInjectedClassNameSpecialization() { Common *CommonPtr = getCommonPtr(); if (!CommonPtr->InjectedClassNameType.isNull()) return CommonPtr->InjectedClassNameType; // C++0x [temp.dep.type]p2: // The template argument list of a primary template is a template argument // list in which the nth template argument has the value of the nth template // parameter of the class template. If the nth template parameter is a // template parameter pack (14.5.3), the nth template argument is a pack // expansion (14.5.3) whose pattern is the name of the template parameter // pack. ASTContext &Context = getASTContext(); TemplateParameterList *Params = getTemplateParameters(); SmallVector<TemplateArgument, 16> TemplateArgs; TemplateArgs.resize(Params->size()); GenerateInjectedTemplateArgs(getASTContext(), Params, TemplateArgs.data()); CommonPtr->InjectedClassNameType = Context.getTemplateSpecializationType(TemplateName(this), &TemplateArgs[0], TemplateArgs.size()); return CommonPtr->InjectedClassNameType; }
void StackColoring::calculateLiveIntervals(unsigned NumSlots) { SmallVector<SlotIndex, 16> Starts; SmallVector<SlotIndex, 16> Finishes; // For each block, find which slots are active within this block // and update the live intervals. for (MachineFunction::iterator MBB = MF->begin(), MBBe = MF->end(); MBB != MBBe; ++MBB) { Starts.clear(); Starts.resize(NumSlots); Finishes.clear(); Finishes.resize(NumSlots); // Create the interval for the basic blocks with lifetime markers in them. for (SmallVectorImpl<MachineInstr*>::const_iterator it = Markers.begin(), e = Markers.end(); it != e; ++it) { const MachineInstr *MI = *it; if (MI->getParent() != MBB) continue; assert((MI->getOpcode() == TargetOpcode::LIFETIME_START || MI->getOpcode() == TargetOpcode::LIFETIME_END) && "Invalid Lifetime marker"); bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START; const MachineOperand &Mo = MI->getOperand(0); int Slot = Mo.getIndex(); assert(Slot >= 0 && "Invalid slot"); SlotIndex ThisIndex = Indexes->getInstructionIndex(MI); if (IsStart) { if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex) Starts[Slot] = ThisIndex; } else { if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex) Finishes[Slot] = ThisIndex; } } // Create the interval of the blocks that we previously found to be 'alive'. BlockLifetimeInfo &MBBLiveness = BlockLiveness[MBB]; for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1; pos = MBBLiveness.LiveIn.find_next(pos)) { Starts[pos] = Indexes->getMBBStartIdx(MBB); } for (int pos = MBBLiveness.LiveOut.find_first(); pos != -1; pos = MBBLiveness.LiveOut.find_next(pos)) { Finishes[pos] = Indexes->getMBBEndIdx(MBB); } for (unsigned i = 0; i < NumSlots; ++i) { assert(Starts[i].isValid() == Finishes[i].isValid() && "Unmatched range"); if (!Starts[i].isValid()) continue; assert(Starts[i] && Finishes[i] && "Invalid interval"); VNInfo *ValNum = Intervals[i]->getValNumInfo(0); SlotIndex S = Starts[i]; SlotIndex F = Finishes[i]; if (S < F) { // We have a single consecutive region. Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum)); } else { // We have two non-consecutive regions. This happens when // LIFETIME_START appears after the LIFETIME_END marker. SlotIndex NewStart = Indexes->getMBBStartIdx(MBB); SlotIndex NewFin = Indexes->getMBBEndIdx(MBB); Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum)); Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum)); } } } }
void MatcherGen:: EmitResultInstructionAsOperand(const TreePatternNode *N, SmallVectorImpl<unsigned> &OutputOps) { Record *Op = N->getOperator(); const CodeGenTarget &CGT = CGP.getTargetInfo(); CodeGenInstruction &II = CGT.getInstruction(Op); const DAGInstruction &Inst = CGP.getInstruction(Op); bool isRoot = N == Pattern.getDstPattern(); // TreeHasOutGlue - True if this tree has glue. bool TreeHasInGlue = false, TreeHasOutGlue = false; if (isRoot) { const TreePatternNode *SrcPat = Pattern.getSrcPattern(); TreeHasInGlue = SrcPat->TreeHasProperty(SDNPOptInGlue, CGP) || SrcPat->TreeHasProperty(SDNPInGlue, CGP); // FIXME2: this is checking the entire pattern, not just the node in // question, doing this just for the root seems like a total hack. TreeHasOutGlue = SrcPat->TreeHasProperty(SDNPOutGlue, CGP); } // NumResults - This is the number of results produced by the instruction in // the "outs" list. unsigned NumResults = Inst.getNumResults(); // Number of operands we know the output instruction must have. If it is // variadic, we could have more operands. unsigned NumFixedOperands = II.Operands.size(); SmallVector<unsigned, 8> InstOps; // Loop over all of the fixed operands of the instruction pattern, emitting // code to fill them all in. The node 'N' usually has number children equal to // the number of input operands of the instruction. However, in cases where // there are predicate operands for an instruction, we need to fill in the // 'execute always' values. Match up the node operands to the instruction // operands to do this. unsigned ChildNo = 0; for (unsigned InstOpNo = NumResults, e = NumFixedOperands; InstOpNo != e; ++InstOpNo) { // Determine what to emit for this operand. Record *OperandNode = II.Operands[InstOpNo].Rec; if (OperandNode->isSubClassOf("OperandWithDefaultOps") && !CGP.getDefaultOperand(OperandNode).DefaultOps.empty()) { // This is a predicate or optional def operand; emit the // 'default ops' operands. const DAGDefaultOperand &DefaultOp = CGP.getDefaultOperand(OperandNode); for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i) EmitResultOperand(DefaultOp.DefaultOps[i].get(), InstOps); continue; } // Otherwise this is a normal operand or a predicate operand without // 'execute always'; emit it. // For operands with multiple sub-operands we may need to emit // multiple child patterns to cover them all. However, ComplexPattern // children may themselves emit multiple MI operands. unsigned NumSubOps = 1; if (OperandNode->isSubClassOf("Operand")) { DagInit *MIOpInfo = OperandNode->getValueAsDag("MIOperandInfo"); if (unsigned NumArgs = MIOpInfo->getNumArgs()) NumSubOps = NumArgs; } unsigned FinalNumOps = InstOps.size() + NumSubOps; while (InstOps.size() < FinalNumOps) { const TreePatternNode *Child = N->getChild(ChildNo); unsigned BeforeAddingNumOps = InstOps.size(); EmitResultOperand(Child, InstOps); assert(InstOps.size() > BeforeAddingNumOps && "Didn't add any operands"); // If the operand is an instruction and it produced multiple results, just // take the first one. if (!Child->isLeaf() && Child->getOperator()->isSubClassOf("Instruction")) InstOps.resize(BeforeAddingNumOps+1); ++ChildNo; } } // If this is a variadic output instruction (i.e. REG_SEQUENCE), we can't // expand suboperands, use default operands, or other features determined from // the CodeGenInstruction after the fixed operands, which were handled // above. Emit the remaining instructions implicitly added by the use for // variable_ops. if (II.Operands.isVariadic) { for (unsigned I = ChildNo, E = N->getNumChildren(); I < E; ++I) EmitResultOperand(N->getChild(I), InstOps); } // If this node has input glue or explicitly specified input physregs, we // need to add chained and glued copyfromreg nodes and materialize the glue // input. if (isRoot && !PhysRegInputs.empty()) { // Emit all of the CopyToReg nodes for the input physical registers. These // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src). for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) AddMatcher(new EmitCopyToRegMatcher(PhysRegInputs[i].second, PhysRegInputs[i].first)); // Even if the node has no other glue inputs, the resultant node must be // glued to the CopyFromReg nodes we just generated. TreeHasInGlue = true; } // Result order: node results, chain, glue // Determine the result types. SmallVector<MVT::SimpleValueType, 4> ResultVTs; for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) ResultVTs.push_back(N->getSimpleType(i)); // If this is the root instruction of a pattern that has physical registers in // its result pattern, add output VTs for them. For example, X86 has: // (set AL, (mul ...)) // This also handles implicit results like: // (implicit EFLAGS) if (isRoot && !Pattern.getDstRegs().empty()) { // If the root came from an implicit def in the instruction handling stuff, // don't re-add it. Record *HandledReg = nullptr; if (II.HasOneImplicitDefWithKnownVT(CGT) != MVT::Other) HandledReg = II.ImplicitDefs[0]; for (Record *Reg : Pattern.getDstRegs()) { if (!Reg->isSubClassOf("Register") || Reg == HandledReg) continue; ResultVTs.push_back(getRegisterValueType(Reg, CGT)); } } // If this is the root of the pattern and the pattern we're matching includes // a node that is variadic, mark the generated node as variadic so that it // gets the excess operands from the input DAG. int NumFixedArityOperands = -1; if (isRoot && Pattern.getSrcPattern()->NodeHasProperty(SDNPVariadic, CGP)) NumFixedArityOperands = Pattern.getSrcPattern()->getNumChildren(); // If this is the root node and multiple matched nodes in the input pattern // have MemRefs in them, have the interpreter collect them and plop them onto // this node. If there is just one node with MemRefs, leave them on that node // even if it is not the root. // // FIXME3: This is actively incorrect for result patterns with multiple // memory-referencing instructions. bool PatternHasMemOperands = Pattern.getSrcPattern()->TreeHasProperty(SDNPMemOperand, CGP); bool NodeHasMemRefs = false; if (PatternHasMemOperands) { unsigned NumNodesThatLoadOrStore = numNodesThatMayLoadOrStore(Pattern.getDstPattern(), CGP); bool NodeIsUniqueLoadOrStore = mayInstNodeLoadOrStore(N, CGP) && NumNodesThatLoadOrStore == 1; NodeHasMemRefs = NodeIsUniqueLoadOrStore || (isRoot && (mayInstNodeLoadOrStore(N, CGP) || NumNodesThatLoadOrStore != 1)); } // Determine whether we need to attach a chain to this node. bool NodeHasChain = false; if (Pattern.getSrcPattern()->TreeHasProperty(SDNPHasChain, CGP)) { // For some instructions, we were able to infer from the pattern whether // they should have a chain. Otherwise, attach the chain to the root. // // FIXME2: This is extremely dubious for several reasons, not the least of // which it gives special status to instructions with patterns that Pat<> // nodes can't duplicate. if (II.hasChain_Inferred) NodeHasChain = II.hasChain; else NodeHasChain = isRoot; // Instructions which load and store from memory should have a chain, // regardless of whether they happen to have a pattern saying so. if (II.hasCtrlDep || II.mayLoad || II.mayStore || II.canFoldAsLoad || II.hasSideEffects) NodeHasChain = true; } assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) && "Node has no result"); AddMatcher(new EmitNodeMatcher(II.Namespace.str()+"::"+II.TheDef->getName().str(), ResultVTs, InstOps, NodeHasChain, TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs, NumFixedArityOperands, NextRecordedOperandNo)); // The non-chain and non-glue results of the newly emitted node get recorded. for (unsigned i = 0, e = ResultVTs.size(); i != e; ++i) { if (ResultVTs[i] == MVT::Other || ResultVTs[i] == MVT::Glue) break; OutputOps.push_back(NextRecordedOperandNo++); } }
/// At this point, we know that this element satisfies the definitive init /// requirements, so we can try to promote loads to enable SSA-based dataflow /// analysis. We know that accesses to this element only access this element, /// cross element accesses have been scalarized. /// /// This returns true if the load has been removed from the program. /// bool AllocOptimize::promoteLoad(SILInstruction *Inst) { // Note that we intentionally don't support forwarding of weak pointers, // because the underlying value may drop be deallocated at any time. We would // have to prove that something in this function is holding the weak value // live across the promoted region and that isn't desired for a stable // diagnostics pass this like one. // We only handle load and copy_addr right now. if (auto CAI = dyn_cast<CopyAddrInst>(Inst)) { // If this is a CopyAddr, verify that the element type is loadable. If not, // we can't explode to a load. if (!CAI->getSrc().getType().isLoadable(Module)) return false; } else if (!isa<LoadInst>(Inst)) return false; // If the box has escaped at this instruction, we can't safely promote the // load. if (hasEscapedAt(Inst)) return false; SILType LoadTy = Inst->getOperand(0).getType().getObjectType(); // If this is a load/copy_addr from a struct field that we want to promote, // compute the access path down to the field so we can determine precise // def/use behavior. unsigned FirstElt = computeSubelement(Inst->getOperand(0), TheMemory); // If this is a load from within an enum projection, we can't promote it since // we don't track subelements in a type that could be changing. if (FirstElt == ~0U) return false; unsigned NumLoadSubElements = getNumSubElements(LoadTy, Module); // Set up the bitvector of elements being demanded by the load. llvm::SmallBitVector RequiredElts(NumMemorySubElements); RequiredElts.set(FirstElt, FirstElt+NumLoadSubElements); SmallVector<std::pair<SILValue, unsigned>, 8> AvailableValues; AvailableValues.resize(NumMemorySubElements); // Find out if we have any available values. If no bits are demanded, we // trivially succeed. This can happen when there is a load of an empty struct. if (NumLoadSubElements != 0) { computeAvailableValues(Inst, RequiredElts, AvailableValues); // If there are no values available at this load point, then we fail to // promote this load and there is nothing to do. bool AnyAvailable = false; for (unsigned i = FirstElt, e = i+NumLoadSubElements; i != e; ++i) if (AvailableValues[i].first.isValid()) { AnyAvailable = true; break; } if (!AnyAvailable) return false; } // Ok, we have some available values. If we have a copy_addr, explode it now, // exposing the load operation within it. Subsequent optimization passes will // see the load and propagate the available values into it. if (auto *CAI = dyn_cast<CopyAddrInst>(Inst)) { explodeCopyAddr(CAI); // This is removing the copy_addr, but explodeCopyAddr takes care of // removing the instruction from Uses for us, so we return false. return false; } // Aggregate together all of the subelements into something that has the same // type as the load did, and emit smaller) loads for any subelements that were // not available. auto NewVal = AggregateAvailableValues(Inst, LoadTy, Inst->getOperand(0), AvailableValues, FirstElt); ++NumLoadPromoted; // Simply replace the load. assert(isa<LoadInst>(Inst)); DEBUG(llvm::dbgs() << " *** Promoting load: " << *Inst << "\n"); DEBUG(llvm::dbgs() << " To value: " << *NewVal.getDef() << "\n"); SILValue(Inst, 0).replaceAllUsesWith(NewVal); SILValue Addr = Inst->getOperand(0); Inst->eraseFromParent(); if (auto *AddrI = dyn_cast<SILInstruction>(Addr)) recursivelyDeleteTriviallyDeadInstructions(AddrI); return true; }
void MatcherGen:: EmitResultInstructionAsOperand(const TreePatternNode *N, SmallVectorImpl<unsigned> &OutputOps) { Record *Op = N->getOperator(); const CodeGenTarget &CGT = CGP.getTargetInfo(); CodeGenInstruction &II = CGT.getInstruction(Op); const DAGInstruction &Inst = CGP.getInstruction(Op); // If we can, get the pattern for the instruction we're generating. We derive // a variety of information from this pattern, such as whether it has a chain. // // FIXME2: This is extremely dubious for several reasons, not the least of // which it gives special status to instructions with patterns that Pat<> // nodes can't duplicate. const TreePatternNode *InstPatNode = GetInstPatternNode(Inst, N); // NodeHasChain - Whether the instruction node we're creating takes chains. bool NodeHasChain = InstPatNode && InstPatNode->TreeHasProperty(SDNPHasChain, CGP); bool isRoot = N == Pattern.getDstPattern(); // TreeHasOutGlue - True if this tree has glue. bool TreeHasInGlue = false, TreeHasOutGlue = false; if (isRoot) { const TreePatternNode *SrcPat = Pattern.getSrcPattern(); TreeHasInGlue = SrcPat->TreeHasProperty(SDNPOptInGlue, CGP) || SrcPat->TreeHasProperty(SDNPInGlue, CGP); // FIXME2: this is checking the entire pattern, not just the node in // question, doing this just for the root seems like a total hack. TreeHasOutGlue = SrcPat->TreeHasProperty(SDNPOutGlue, CGP); } // NumResults - This is the number of results produced by the instruction in // the "outs" list. unsigned NumResults = Inst.getNumResults(); // Loop over all of the operands of the instruction pattern, emitting code // to fill them all in. The node 'N' usually has number children equal to // the number of input operands of the instruction. However, in cases // where there are predicate operands for an instruction, we need to fill // in the 'execute always' values. Match up the node operands to the // instruction operands to do this. SmallVector<unsigned, 8> InstOps; for (unsigned ChildNo = 0, InstOpNo = NumResults, e = II.Operands.size(); InstOpNo != e; ++InstOpNo) { // Determine what to emit for this operand. Record *OperandNode = II.Operands[InstOpNo].Rec; if ((OperandNode->isSubClassOf("PredicateOperand") || OperandNode->isSubClassOf("OptionalDefOperand")) && !CGP.getDefaultOperand(OperandNode).DefaultOps.empty()) { // This is a predicate or optional def operand; emit the // 'default ops' operands. const DAGDefaultOperand &DefaultOp = CGP.getDefaultOperand(OperandNode); for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i) EmitResultOperand(DefaultOp.DefaultOps[i], InstOps); continue; } const TreePatternNode *Child = N->getChild(ChildNo); // Otherwise this is a normal operand or a predicate operand without // 'execute always'; emit it. unsigned BeforeAddingNumOps = InstOps.size(); EmitResultOperand(Child, InstOps); assert(InstOps.size() > BeforeAddingNumOps && "Didn't add any operands"); // If the operand is an instruction and it produced multiple results, just // take the first one. if (!Child->isLeaf() && Child->getOperator()->isSubClassOf("Instruction")) InstOps.resize(BeforeAddingNumOps+1); ++ChildNo; } // If this node has input glue or explicitly specified input physregs, we // need to add chained and glued copyfromreg nodes and materialize the glue // input. if (isRoot && !PhysRegInputs.empty()) { // Emit all of the CopyToReg nodes for the input physical registers. These // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src). for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) AddMatcher(new EmitCopyToRegMatcher(PhysRegInputs[i].second, PhysRegInputs[i].first)); // Even if the node has no other glue inputs, the resultant node must be // glued to the CopyFromReg nodes we just generated. TreeHasInGlue = true; } // Result order: node results, chain, glue // Determine the result types. SmallVector<MVT::SimpleValueType, 4> ResultVTs; for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) ResultVTs.push_back(N->getType(i)); // If this is the root instruction of a pattern that has physical registers in // its result pattern, add output VTs for them. For example, X86 has: // (set AL, (mul ...)) // This also handles implicit results like: // (implicit EFLAGS) if (isRoot && !Pattern.getDstRegs().empty()) { // If the root came from an implicit def in the instruction handling stuff, // don't re-add it. Record *HandledReg = 0; if (II.HasOneImplicitDefWithKnownVT(CGT) != MVT::Other) HandledReg = II.ImplicitDefs[0]; for (unsigned i = 0; i != Pattern.getDstRegs().size(); ++i) { Record *Reg = Pattern.getDstRegs()[i]; if (!Reg->isSubClassOf("Register") || Reg == HandledReg) continue; ResultVTs.push_back(getRegisterValueType(Reg, CGT)); } } // If this is the root of the pattern and the pattern we're matching includes // a node that is variadic, mark the generated node as variadic so that it // gets the excess operands from the input DAG. int NumFixedArityOperands = -1; if (isRoot && (Pattern.getSrcPattern()->NodeHasProperty(SDNPVariadic, CGP))) NumFixedArityOperands = Pattern.getSrcPattern()->getNumChildren(); // If this is the root node and multiple matched nodes in the input pattern // have MemRefs in them, have the interpreter collect them and plop them onto // this node. If there is just one node with MemRefs, leave them on that node // even if it is not the root. // // FIXME3: This is actively incorrect for result patterns with multiple // memory-referencing instructions. bool PatternHasMemOperands = Pattern.getSrcPattern()->TreeHasProperty(SDNPMemOperand, CGP); bool NodeHasMemRefs = false; if (PatternHasMemOperands) { unsigned NumNodesThatLoadOrStore = numNodesThatMayLoadOrStore(Pattern.getDstPattern(), CGP); bool NodeIsUniqueLoadOrStore = mayInstNodeLoadOrStore(N, CGP) && NumNodesThatLoadOrStore == 1; NodeHasMemRefs = NodeIsUniqueLoadOrStore || (isRoot && (mayInstNodeLoadOrStore(N, CGP) || NumNodesThatLoadOrStore != 1)); } assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) && "Node has no result"); AddMatcher(new EmitNodeMatcher(II.Namespace+"::"+II.TheDef->getName(), ResultVTs.data(), ResultVTs.size(), InstOps.data(), InstOps.size(), NodeHasChain, TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs, NumFixedArityOperands, NextRecordedOperandNo)); // The non-chain and non-glue results of the newly emitted node get recorded. for (unsigned i = 0, e = ResultVTs.size(); i != e; ++i) { if (ResultVTs[i] == MVT::Other || ResultVTs[i] == MVT::Glue) break; OutputOps.push_back(NextRecordedOperandNo++); } }