void AMDGPUTargetInfo::fillValidCPUList( SmallVectorImpl<StringRef> &Values) const { if (isAMDGCN(getTriple())) llvm::for_each(AMDGCNGPUs, [&Values](const GPUInfo &GPU) { Values.emplace_back(GPU.Name);}); else llvm::for_each(R600GPUs, [&Values](const GPUInfo &GPU) { Values.emplace_back(GPU.Name);}); }
// TODO: This should really be an operation on type lowering. void SILBuilder::emitDestructureValueOperation( SILLocation loc, SILValue v, SmallVectorImpl<SILValue> &results) { // Once destructure is allowed everywhere, remove the projection code. // If we do not have a tuple or a struct, add to our results list and return. SILType type = v->getType(); if (!(type.is<TupleType>() || type.getStructOrBoundGenericStruct())) { results.emplace_back(v); return; } // Otherwise, we want to destructure add the destructure and return. if (getFunction().hasOwnership()) { auto *i = emitDestructureValueOperation(loc, v); copy(i->getResults(), std::back_inserter(results)); return; } // In non qualified ownership SIL, drop back to using projection code. SmallVector<Projection, 16> projections; Projection::getFirstLevelProjections(v->getType(), getModule(), projections); transform(projections, std::back_inserter(results), [&](const Projection &p) -> SILValue { return p.createObjectProjection(*this, loc, v).get(); }); }
void Scheduler::promoteToReadyQueue(SmallVectorImpl<InstRef> &Ready) { // Scan the set of waiting instructions and promote them to the // ready queue if operands are all ready. for (auto I = WaitQueue.begin(), E = WaitQueue.end(); I != E;) { const unsigned IID = I->first; Instruction *IS = I->second; // Check if this instruction is now ready. In case, force // a transition in state using method 'update()'. IS->update(); const InstrDesc &Desc = IS->getDesc(); bool IsMemOp = Desc.MayLoad || Desc.MayStore; if (!IS->isReady() || (IsMemOp && !LSU->isReady({IID, IS}))) { ++I; continue; } Ready.emplace_back(IID, IS); ReadyQueue[IID] = IS; auto ToRemove = I; ++I; WaitQueue.erase(ToRemove); } }
void ResourceManager::issueInstruction( const InstrDesc &Desc, SmallVectorImpl<std::pair<ResourceRef, double>> &Pipes) { for (const std::pair<uint64_t, ResourceUsage> &R : Desc.Resources) { const CycleSegment &CS = R.second.CS; if (!CS.size()) { releaseResource(R.first); continue; } assert(CS.begin() == 0 && "Invalid {Start, End} cycles!"); if (!R.second.isReserved()) { ResourceRef Pipe = selectPipe(R.first); use(Pipe); BusyResources[Pipe] += CS.size(); // Replace the resource mask with a valid processor resource index. const ResourceState &RS = *Resources[Pipe.first]; Pipe.first = RS.getProcResourceID(); Pipes.emplace_back( std::pair<ResourceRef, double>(Pipe, static_cast<double>(CS.size()))); } else { assert((countPopulation(R.first) > 1) && "Expected a group!"); // Mark this group as reserved. assert(R.second.isReserved()); reserveResource(R.first); BusyResources[ResourceRef(R.first, R.first)] += CS.size(); } } }
void CounterExpressionBuilder::extractTerms(Counter C, int Factor, SmallVectorImpl<Term> &Terms) { switch (C.getKind()) { case Counter::Zero: break; case Counter::CounterValueReference: Terms.emplace_back(C.getCounterID(), Factor); break; case Counter::Expression: const auto &E = Expressions[C.getExpressionID()]; extractTerms(E.LHS, Factor, Terms); extractTerms( E.RHS, E.Kind == CounterExpression::Subtract ? -Factor : Factor, Terms); break; } }
// TODO: Can we put this on type lowering? It would take a little bit of work // since we would need to be able to handle aggregate trivial types which is not // represented today in TypeLowering. void SILBuilder::emitDestructureAddressOperation( SILLocation loc, SILValue v, SmallVectorImpl<SILValue> &results) { // If we do not have a tuple or a struct, add to our results list. SILType type = v->getType(); if (!(type.is<TupleType>() || type.getStructOrBoundGenericStruct())) { results.emplace_back(v); return; } SmallVector<Projection, 16> projections; Projection::getFirstLevelProjections(v->getType(), getModule(), projections); transform(projections, std::back_inserter(results), [&](const Projection &p) -> SILValue { return p.createAddressProjection(*this, loc, v).get(); }); }
bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, SplitArgTy PerformArgSplit) const { const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); LLVMContext &Context = OrigArg.Ty->getContext(); SmallVector<EVT, 4> SplitVTs; SmallVector<uint64_t, 4> Offsets; ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); if (SplitVTs.size() != 1) { // TODO: support struct/array split return false; } EVT VT = SplitVTs[0]; unsigned NumParts = TLI.getNumRegisters(Context, VT); if (NumParts == 1) { // replace the original type ( pointer -> GPR ). SplitArgs.emplace_back(OrigArg.Reg, VT.getTypeForEVT(Context), OrigArg.Flags, OrigArg.IsFixed); return true; } SmallVector<unsigned, 8> SplitRegs; EVT PartVT = TLI.getRegisterType(Context, VT); Type *PartTy = PartVT.getTypeForEVT(Context); for (unsigned i = 0; i < NumParts; ++i) { ArgInfo Info = ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), PartTy, OrigArg.Flags}; SplitArgs.push_back(Info); SplitRegs.push_back(Info.Reg); } PerformArgSplit(SplitRegs); return true; }
void AArch64CallLowering::splitToValueTypes( const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv, const SplitArgTy &PerformArgSplit) const { const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>(); LLVMContext &Ctx = OrigArg.Ty->getContext(); if (OrigArg.Ty->isVoidTy()) return; SmallVector<EVT, 4> SplitVTs; SmallVector<uint64_t, 4> Offsets; ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); if (SplitVTs.size() == 1) { // No splitting to do, but we want to replace the original type (e.g. [1 x // double] -> double). SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), OrigArg.Flags, OrigArg.IsFixed); return; } unsigned FirstRegIdx = SplitArgs.size(); bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( OrigArg.Ty, CallConv, false); for (auto SplitVT : SplitVTs) { Type *SplitTy = SplitVT.getTypeForEVT(Ctx); SplitArgs.push_back( ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)), SplitTy, OrigArg.Flags, OrigArg.IsFixed}); if (NeedsRegBlock) SplitArgs.back().Flags.setInConsecutiveRegs(); } SplitArgs.back().Flags.setInConsecutiveRegsLast(); for (unsigned i = 0; i < Offsets.size(); ++i) PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8); }
void AArch64CallLowering::splitToValueTypes( const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs, const DataLayout &DL, MachineRegisterInfo &MRI, const SplitArgTy &PerformArgSplit) const { const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>(); LLVMContext &Ctx = OrigArg.Ty->getContext(); SmallVector<EVT, 4> SplitVTs; SmallVector<uint64_t, 4> Offsets; ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); if (SplitVTs.size() == 1) { // No splitting to do, but we want to replace the original type (e.g. [1 x // double] -> double). SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), OrigArg.Flags, OrigArg.IsFixed); return; } unsigned FirstRegIdx = SplitArgs.size(); for (auto SplitVT : SplitVTs) { // FIXME: set split flags if they're actually used (e.g. i128 on AAPCS). Type *SplitTy = SplitVT.getTypeForEVT(Ctx); SplitArgs.push_back( ArgInfo{MRI.createGenericVirtualRegister(LLT{*SplitTy, DL}), SplitTy, OrigArg.Flags, OrigArg.IsFixed}); } SmallVector<uint64_t, 4> BitOffsets; for (auto Offset : Offsets) BitOffsets.push_back(Offset * 8); SmallVector<unsigned, 8> SplitRegs; for (auto I = &SplitArgs[FirstRegIdx]; I != SplitArgs.end(); ++I) SplitRegs.push_back(I->Reg); PerformArgSplit(SplitRegs, BitOffsets); }
bool GuardWideningImpl::combineRangeChecks( SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks, SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) { unsigned OldCount = Checks.size(); while (!Checks.empty()) { // Pick all of the range checks with a specific base and length, and try to // merge them. Value *CurrentBase = Checks.front().getBase(); Value *CurrentLength = Checks.front().getLength(); SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks; auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) { return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength; }; copy_if(Checks, std::back_inserter(CurrentChecks), IsCurrentCheck); Checks.erase(remove_if(Checks, IsCurrentCheck), Checks.end()); assert(CurrentChecks.size() != 0 && "We know we have at least one!"); if (CurrentChecks.size() < 3) { RangeChecksOut.insert(RangeChecksOut.end(), CurrentChecks.begin(), CurrentChecks.end()); continue; } // CurrentChecks.size() will typically be 3 here, but so far there has been // no need to hard-code that fact. std::sort(CurrentChecks.begin(), CurrentChecks.end(), [&](const GuardWideningImpl::RangeCheck &LHS, const GuardWideningImpl::RangeCheck &RHS) { return LHS.getOffsetValue().slt(RHS.getOffsetValue()); }); // Note: std::sort should not invalidate the ChecksStart iterator. ConstantInt *MinOffset = CurrentChecks.front().getOffset(), *MaxOffset = CurrentChecks.back().getOffset(); unsigned BitWidth = MaxOffset->getValue().getBitWidth(); if ((MaxOffset->getValue() - MinOffset->getValue()) .ugt(APInt::getSignedMinValue(BitWidth))) return false; APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue(); const APInt &HighOffset = MaxOffset->getValue(); auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) { return (HighOffset - RC.getOffsetValue()).ult(MaxDiff); }; if (MaxDiff.isMinValue() || !std::all_of(std::next(CurrentChecks.begin()), CurrentChecks.end(), OffsetOK)) return false; // We have a series of f+1 checks as: // // I+k_0 u< L ... Chk_0 // I_k_1 u< L ... Chk_1 // ... // I_k_f u< L ... Chk_(f+1) // // with forall i in [0,f): k_f-k_i u< k_f-k_0 ... Precond_0 // k_f-k_0 u< INT_MIN+k_f ... Precond_1 // k_f != k_0 ... Precond_2 // // Claim: // Chk_0 AND Chk_(f+1) implies all the other checks // // Informal proof sketch: // // We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap // (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and // thus I+k_f is the greatest unsigned value in that range. // // This combined with Ckh_(f+1) shows that everything in that range is u< L. // Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1) // lie in [I+k_0,I+k_f], this proving our claim. // // To see that [I+k_0,I+k_f] is not a wrapping range, note that there are // two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal // since k_0 != k_f). In the former case, [I+k_0,I+k_f] is not a wrapping // range by definition, and the latter case is impossible: // // 0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1) // xxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx // // For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted // with 'x' above) to be at least >u INT_MIN. RangeChecksOut.emplace_back(CurrentChecks.front()); RangeChecksOut.emplace_back(CurrentChecks.back()); } assert(RangeChecksOut.size() <= OldCount && "We pessimized!"); return RangeChecksOut.size() != OldCount; }
/// AnalyzeAsmString - Analyze the asm string of the current asm, decomposing /// it into pieces. If the asm string is erroneous, emit errors and return /// true, otherwise return false. unsigned GCCAsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces, const ASTContext &C, unsigned &DiagOffs) const { StringRef Str = getAsmString()->getString(); const char *StrStart = Str.begin(); const char *StrEnd = Str.end(); const char *CurPtr = StrStart; // "Simple" inline asms have no constraints or operands, just convert the asm // string to escape $'s. if (isSimple()) { std::string Result; for (; CurPtr != StrEnd; ++CurPtr) { switch (*CurPtr) { case '$': Result += "$$"; break; default: Result += *CurPtr; break; } } Pieces.push_back(AsmStringPiece(Result)); return 0; } // CurStringPiece - The current string that we are building up as we scan the // asm string. std::string CurStringPiece; bool HasVariants = !C.getTargetInfo().hasNoAsmVariants(); while (1) { // Done with the string? if (CurPtr == StrEnd) { if (!CurStringPiece.empty()) Pieces.push_back(AsmStringPiece(CurStringPiece)); return 0; } char CurChar = *CurPtr++; switch (CurChar) { case '$': CurStringPiece += "$$"; continue; case '{': CurStringPiece += (HasVariants ? "$(" : "{"); continue; case '|': CurStringPiece += (HasVariants ? "$|" : "|"); continue; case '}': CurStringPiece += (HasVariants ? "$)" : "}"); continue; case '%': break; default: CurStringPiece += CurChar; continue; } // Escaped "%" character in asm string. if (CurPtr == StrEnd) { // % at end of string is invalid (no escape). DiagOffs = CurPtr-StrStart-1; return diag::err_asm_invalid_escape; } char EscapedChar = *CurPtr++; if (EscapedChar == '%') { // %% -> % // Escaped percentage sign. CurStringPiece += '%'; continue; } if (EscapedChar == '=') { // %= -> Generate an unique ID. CurStringPiece += "${:uid}"; continue; } // Otherwise, we have an operand. If we have accumulated a string so far, // add it to the Pieces list. if (!CurStringPiece.empty()) { Pieces.push_back(AsmStringPiece(CurStringPiece)); CurStringPiece.clear(); } // Handle operands that have asmSymbolicName (e.g., %x[foo]) and those that // don't (e.g., %x4). 'x' following the '%' is the constraint modifier. const char *Begin = CurPtr - 1; // Points to the character following '%'. const char *Percent = Begin - 1; // Points to '%'. if (isLetter(EscapedChar)) { if (CurPtr == StrEnd) { // Premature end. DiagOffs = CurPtr-StrStart-1; return diag::err_asm_invalid_escape; } EscapedChar = *CurPtr++; } const TargetInfo &TI = C.getTargetInfo(); const SourceManager &SM = C.getSourceManager(); const LangOptions &LO = C.getLangOpts(); // Handle operands that don't have asmSymbolicName (e.g., %x4). if (isDigit(EscapedChar)) { // %n - Assembler operand n unsigned N = 0; --CurPtr; while (CurPtr != StrEnd && isDigit(*CurPtr)) N = N*10 + ((*CurPtr++)-'0'); unsigned NumOperands = getNumOutputs() + getNumPlusOperands() + getNumInputs(); if (N >= NumOperands) { DiagOffs = CurPtr-StrStart-1; return diag::err_asm_invalid_operand_number; } // Str contains "x4" (Operand without the leading %). std::string Str(Begin, CurPtr - Begin); // (BeginLoc, EndLoc) represents the range of the operand we are currently // processing. Unlike Str, the range includes the leading '%'. SourceLocation BeginLoc = getAsmString()->getLocationOfByte(Percent - StrStart, SM, LO, TI); SourceLocation EndLoc = getAsmString()->getLocationOfByte(CurPtr - StrStart, SM, LO, TI); Pieces.emplace_back(N, std::move(Str), BeginLoc, EndLoc); continue; } // Handle operands that have asmSymbolicName (e.g., %x[foo]). if (EscapedChar == '[') { DiagOffs = CurPtr-StrStart-1; // Find the ']'. const char *NameEnd = (const char*)memchr(CurPtr, ']', StrEnd-CurPtr); if (NameEnd == nullptr) return diag::err_asm_unterminated_symbolic_operand_name; if (NameEnd == CurPtr) return diag::err_asm_empty_symbolic_operand_name; StringRef SymbolicName(CurPtr, NameEnd - CurPtr); int N = getNamedOperand(SymbolicName); if (N == -1) { // Verify that an operand with that name exists. DiagOffs = CurPtr-StrStart; return diag::err_asm_unknown_symbolic_operand_name; } // Str contains "x[foo]" (Operand without the leading %). std::string Str(Begin, NameEnd + 1 - Begin); // (BeginLoc, EndLoc) represents the range of the operand we are currently // processing. Unlike Str, the range includes the leading '%'. SourceLocation BeginLoc = getAsmString()->getLocationOfByte(Percent - StrStart, SM, LO, TI); SourceLocation EndLoc = getAsmString()->getLocationOfByte(NameEnd + 1 - StrStart, SM, LO, TI); Pieces.emplace_back(N, std::move(Str), BeginLoc, EndLoc); CurPtr = NameEnd+1; continue; } DiagOffs = CurPtr-StrStart-1; return diag::err_asm_invalid_escape; } }
bool SILValueOwnershipChecker::gatherUsers( SmallVectorImpl<BranchPropagatedUser> &lifetimeEndingUsers, SmallVectorImpl<BranchPropagatedUser> &nonLifetimeEndingUsers, SmallVectorImpl<BranchPropagatedUser> &implicitRegularUsers) { // See if Value is guaranteed. If we are guaranteed and not forwarding, then // we need to look through subobject uses for more uses. Otherwise, if we are // forwarding, we do not create any lifetime ending users/non lifetime ending // users since we verify against our base. auto ownershipKind = value.getOwnershipKind(); bool isGuaranteed = ownershipKind == ValueOwnershipKind::Guaranteed; bool isOwned = ownershipKind == ValueOwnershipKind::Owned; if (isGuaranteed && isGuaranteedForwardingValue(value)) return true; // Then gather up our initial list of users. SmallVector<Operand *, 8> users; std::copy(value->use_begin(), value->use_end(), std::back_inserter(users)); auto addCondBranchToList = [](SmallVectorImpl<BranchPropagatedUser> &list, CondBranchInst *cbi, unsigned operandIndex) { if (cbi->isConditionOperandIndex(operandIndex)) { list.emplace_back(cbi); return; } bool isTrueOperand = cbi->isTrueOperandIndex(operandIndex); list.emplace_back(cbi, isTrueOperand ? CondBranchInst::TrueIdx : CondBranchInst::FalseIdx); }; bool foundError = false; while (!users.empty()) { Operand *op = users.pop_back_val(); SILInstruction *user = op->getUser(); // If this op is a type dependent operand, skip it. It is not interesting // from an ownership perspective. if (user->isTypeDependentOperand(*op)) continue; bool isGuaranteedSubValue = false; if (isGuaranteed && isGuaranteedForwardingInst(op->getUser())) { isGuaranteedSubValue = true; } auto opOwnershipKindMap = op->getOwnershipKindMap(isGuaranteedSubValue); // If our ownership kind doesn't match, track that we found an error, emit // an error message optionally and then continue. if (!opOwnershipKindMap.canAcceptKind(ownershipKind)) { foundError = true; // If we did not support /any/ ownership kind, it means that we found a // conflicting answer so the kind map that was returned is the empty // map. Put out a more specific error here. if (!opOwnershipKindMap.data.any()) { handleError([&]() { llvm::errs() << "Function: '" << user->getFunction()->getName() << "'\n" << "Ill-formed SIL! Unable to compute ownership kind " "map for user?!\n" << "For terminator users, check that successors have " "compatible ownership kinds.\n" << "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n' << "Conv: " << ownershipKind << "\n\n"; }); continue; } handleError([&]() { llvm::errs() << "Function: '" << user->getFunction()->getName() << "'\n" << "Have operand with incompatible ownership?!\n" << "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n' << "Conv: " << ownershipKind << '\n' << "OwnershipMap:\n" << opOwnershipKindMap << '\n'; }); continue; } auto lifetimeConstraint = opOwnershipKindMap.getLifetimeConstraint(ownershipKind); if (lifetimeConstraint == UseLifetimeConstraint::MustBeInvalidated) { LLVM_DEBUG(llvm::dbgs() << " Lifetime Ending User: "******" Regular User: "******"Our value is guaranteed and this is a forwarding instruction. " "Should have guaranteed ownership as well."); copy(result->getUses(), std::back_inserter(users)); } continue; } assert(user->getResults().empty()); auto *ti = dyn_cast<TermInst>(user); if (!ti) { continue; } // Otherwise if we have a terminator, add any as uses any end_borrow to // ensure that the subscope is completely enclsed within the super scope. We // require all of our arguments to be either trivial or guaranteed. for (auto &succ : ti->getSuccessors()) { auto *succBlock = succ.getBB(); // If we do not have any arguments, then continue. if (succBlock->args_empty()) continue; // Otherwise, make sure that all arguments are trivial or guaranteed. If // we fail, emit an error. // // TODO: We could ignore this error and emit a more specific error on the // actual terminator. for (auto *succArg : succBlock->getPhiArguments()) { // *NOTE* We do not emit an error here since we want to allow for more // specific errors to be found during use_verification. // // TODO: Add a flag that associates the terminator instruction with // needing to be verified. If it isn't verified appropriately, assert // when the verifier is destroyed. auto succArgOwnershipKind = succArg->getOwnershipKind(); if (!succArgOwnershipKind.isCompatibleWith(ownershipKind)) { // This is where the error would go. continue; } // If we have an any value, just continue. if (succArgOwnershipKind == ValueOwnershipKind::Any) continue; // Otherwise add all end_borrow users for this BBArg to the // implicit regular user list. We know that BBArg must be // completely joint post-dominated by these users, so we use // them to ensure that all of BBArg's uses are completely // enclosed within the end_borrow of this argument. for (auto *op : succArg->getUses()) { if (auto *ebi = dyn_cast<EndBorrowInst>(op->getUser())) { implicitRegularUsers.push_back(ebi); } } } } } // Return true if we did not have an error and false if we did find an error. // // The reason why we use this extra variable is to make sure that when we are // testing, we print out all mismatching pairs rather than just the first. return !foundError; }
/// Analyze MBB to check if its terminating branch can be turned into an /// implicit null check. If yes, append a description of the said null check to /// NullCheckList and return true, else return false. bool ImplicitNullChecks::analyzeBlockForNullChecks( MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) { typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate; MDNode *BranchMD = nullptr; if (auto *BB = MBB.getBasicBlock()) BranchMD = BB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit); if (!BranchMD) return false; MachineBranchPredicate MBP; if (TII->analyzeBranchPredicate(MBB, MBP, true)) return false; // Is the predicate comparing an integer to zero? if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 && (MBP.Predicate == MachineBranchPredicate::PRED_NE || MBP.Predicate == MachineBranchPredicate::PRED_EQ))) return false; // If we cannot erase the test instruction itself, then making the null check // implicit does not buy us much. if (!MBP.SingleUseCondition) return false; MachineBasicBlock *NotNullSucc, *NullSucc; if (MBP.Predicate == MachineBranchPredicate::PRED_NE) { NotNullSucc = MBP.TrueDest; NullSucc = MBP.FalseDest; } else { NotNullSucc = MBP.FalseDest; NullSucc = MBP.TrueDest; } // We handle the simplest case for now. We can potentially do better by using // the machine dominator tree. if (NotNullSucc->pred_size() != 1) return false; // Starting with a code fragment like: // // test %RAX, %RAX // jne LblNotNull // // LblNull: // callq throw_NullPointerException // // LblNotNull: // Inst0 // Inst1 // ... // Def = Load (%RAX + <offset>) // ... // // // we want to end up with // // Def = FaultingLoad (%RAX + <offset>), LblNull // jmp LblNotNull ;; explicit or fallthrough // // LblNotNull: // Inst0 // Inst1 // ... // // LblNull: // callq throw_NullPointerException // // // To see why this is legal, consider the two possibilities: // // 1. %RAX is null: since we constrain <offset> to be less than PageSize, the // load instruction dereferences the null page, causing a segmentation // fault. // // 2. %RAX is not null: in this case we know that the load cannot fault, as // otherwise the load would've faulted in the original program too and the // original program would've been undefined. // // This reasoning cannot be extended to justify hoisting through arbitrary // control flow. For instance, in the example below (in pseudo-C) // // if (ptr == null) { throw_npe(); unreachable; } // if (some_cond) { return 42; } // v = ptr->field; // LD // ... // // we cannot (without code duplication) use the load marked "LD" to null check // ptr -- clause (2) above does not apply in this case. In the above program // the safety of ptr->field can be dependent on some_cond; and, for instance, // ptr could be some non-null invalid reference that never gets loaded from // because some_cond is always true. const unsigned PointerReg = MBP.LHS.getReg(); SmallVector<MachineInstr *, 8> InstsSeenSoFar; for (auto &MI : *NotNullSucc) { if (!canHandle(&MI) || InstsSeenSoFar.size() >= MaxInstsToConsider) return false; MachineInstr *Dependence; SuitabilityResult SR = isSuitableMemoryOp(MI, PointerReg, InstsSeenSoFar); if (SR == SR_Impossible) return false; if (SR == SR_Suitable && canHoistInst(&MI, PointerReg, InstsSeenSoFar, NullSucc, Dependence)) { NullCheckList.emplace_back(&MI, MBP.ConditionDef, &MBB, NotNullSucc, NullSucc, Dependence); return true; } InstsSeenSoFar.push_back(&MI); } return false; }
/// Analyze MBB to check if its terminating branch can be turned into an /// implicit null check. If yes, append a description of the said null check to /// NullCheckList and return true, else return false. bool ImplicitNullChecks::analyzeBlockForNullChecks( MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) { typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate; MDNode *BranchMD = nullptr; if (auto *BB = MBB.getBasicBlock()) BranchMD = BB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit); if (!BranchMD) return false; MachineBranchPredicate MBP; if (TII->AnalyzeBranchPredicate(MBB, MBP, true)) return false; // Is the predicate comparing an integer to zero? if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 && (MBP.Predicate == MachineBranchPredicate::PRED_NE || MBP.Predicate == MachineBranchPredicate::PRED_EQ))) return false; // If we cannot erase the test instruction itself, then making the null check // implicit does not buy us much. if (!MBP.SingleUseCondition) return false; MachineBasicBlock *NotNullSucc, *NullSucc; if (MBP.Predicate == MachineBranchPredicate::PRED_NE) { NotNullSucc = MBP.TrueDest; NullSucc = MBP.FalseDest; } else { NotNullSucc = MBP.FalseDest; NullSucc = MBP.TrueDest; } // We handle the simplest case for now. We can potentially do better by using // the machine dominator tree. if (NotNullSucc->pred_size() != 1) return false; // Starting with a code fragment like: // // test %RAX, %RAX // jne LblNotNull // // LblNull: // callq throw_NullPointerException // // LblNotNull: // Inst0 // Inst1 // ... // Def = Load (%RAX + <offset>) // ... // // // we want to end up with // // Def = FaultingLoad (%RAX + <offset>), LblNull // jmp LblNotNull ;; explicit or fallthrough // // LblNotNull: // Inst0 // Inst1 // ... // // LblNull: // callq throw_NullPointerException // // // To see why this is legal, consider the two possibilities: // // 1. %RAX is null: since we constrain <offset> to be less than PageSize, the // load instruction dereferences the null page, causing a segmentation // fault. // // 2. %RAX is not null: in this case we know that the load cannot fault, as // otherwise the load would've faulted in the original program too and the // original program would've been undefined. // // This reasoning cannot be extended to justify hoisting through arbitrary // control flow. For instance, in the example below (in pseudo-C) // // if (ptr == null) { throw_npe(); unreachable; } // if (some_cond) { return 42; } // v = ptr->field; // LD // ... // // we cannot (without code duplication) use the load marked "LD" to null check // ptr -- clause (2) above does not apply in this case. In the above program // the safety of ptr->field can be dependent on some_cond; and, for instance, // ptr could be some non-null invalid reference that never gets loaded from // because some_cond is always true. unsigned PointerReg = MBP.LHS.getReg(); HazardDetector HD(*TRI, *AA); for (auto MII = NotNullSucc->begin(), MIE = NotNullSucc->end(); MII != MIE; ++MII) { MachineInstr *MI = &*MII; unsigned BaseReg; int64_t Offset; MachineInstr *Dependency = nullptr; if (TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI)) if (MI->mayLoad() && !MI->isPredicable() && BaseReg == PointerReg && Offset < PageSize && MI->getDesc().getNumDefs() <= 1 && HD.isSafeToHoist(MI, Dependency)) { auto DependencyOperandIsOk = [&](MachineOperand &MO) { assert(!(MO.isReg() && MO.isUse()) && "No transitive dependendencies please!"); if (!MO.isReg() || !MO.getReg() || !MO.isDef()) return true; // Make sure that we won't clobber any live ins to the sibling block // by hoisting Dependency. For instance, we can't hoist INST to // before the null check (even if it safe, and does not violate any // dependencies in the non_null_block) if %rdx is live in to // _null_block. // // test %rcx, %rcx // je _null_block // _non_null_block: // %rdx<def> = INST // ... if (AnyAliasLiveIn(TRI, NullSucc, MO.getReg())) return false; // Make sure Dependency isn't re-defining the base register. Then we // won't get the memory operation on the address we want. if (TRI->regsOverlap(MO.getReg(), BaseReg)) return false; return true; }; bool DependencyOperandsAreOk = !Dependency || all_of(Dependency->operands(), DependencyOperandIsOk); if (DependencyOperandsAreOk) { NullCheckList.emplace_back(MI, MBP.ConditionDef, &MBB, NotNullSucc, NullSucc, Dependency); return true; } } HD.rememberInstruction(MI); if (HD.isClobbered()) return false; } return false; }
void IDFCalculator<NodeTy>::calculate( SmallVectorImpl<BasicBlock *> &PHIBlocks) { // Use a priority queue keyed on dominator tree level so that inserted nodes // are handled from the bottom of the dominator tree upwards. typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair; typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>, less_second> IDFPriorityQueue; IDFPriorityQueue PQ; for (BasicBlock *BB : *DefBlocks) { if (DomTreeNode *Node = DT.getNode(BB)) PQ.push({Node, Node->getLevel()}); } SmallVector<DomTreeNode *, 32> Worklist; SmallPtrSet<DomTreeNode *, 32> VisitedPQ; SmallPtrSet<DomTreeNode *, 32> VisitedWorklist; while (!PQ.empty()) { DomTreeNodePair RootPair = PQ.top(); PQ.pop(); DomTreeNode *Root = RootPair.first; unsigned RootLevel = RootPair.second; // Walk all dominator tree children of Root, inspecting their CFG edges with // targets elsewhere on the dominator tree. Only targets whose level is at // most Root's level are added to the iterated dominance frontier of the // definition set. Worklist.clear(); Worklist.push_back(Root); VisitedWorklist.insert(Root); while (!Worklist.empty()) { DomTreeNode *Node = Worklist.pop_back_val(); BasicBlock *BB = Node->getBlock(); // Succ is the successor in the direction we are calculating IDF, so it is // successor for IDF, and predecessor for Reverse IDF. for (auto *Succ : children<NodeTy>(BB)) { DomTreeNode *SuccNode = DT.getNode(Succ); // Quickly skip all CFG edges that are also dominator tree edges instead // of catching them below. if (SuccNode->getIDom() == Node) continue; const unsigned SuccLevel = SuccNode->getLevel(); if (SuccLevel > RootLevel) continue; if (!VisitedPQ.insert(SuccNode).second) continue; BasicBlock *SuccBB = SuccNode->getBlock(); if (useLiveIn && !LiveInBlocks->count(SuccBB)) continue; PHIBlocks.emplace_back(SuccBB); if (!DefBlocks->count(SuccBB)) PQ.push(std::make_pair(SuccNode, SuccLevel)); } for (auto DomChild : *Node) { if (VisitedWorklist.insert(DomChild).second) Worklist.push_back(DomChild); } } } }
/// Analyze MBB to check if its terminating branch can be turned into an /// implicit null check. If yes, append a description of the said null check to /// NullCheckList and return true, else return false. bool ImplicitNullChecks::analyzeBlockForNullChecks( MachineBasicBlock &MBB, SmallVectorImpl<NullCheck> &NullCheckList) { typedef TargetInstrInfo::MachineBranchPredicate MachineBranchPredicate; MDNode *BranchMD = MBB.getBasicBlock() ? MBB.getBasicBlock()->getTerminator()->getMetadata(LLVMContext::MD_make_implicit) : nullptr; if (!BranchMD) return false; MachineBranchPredicate MBP; if (TII->AnalyzeBranchPredicate(MBB, MBP, true)) return false; // Is the predicate comparing an integer to zero? if (!(MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 && (MBP.Predicate == MachineBranchPredicate::PRED_NE || MBP.Predicate == MachineBranchPredicate::PRED_EQ))) return false; // If we cannot erase the test instruction itself, then making the null check // implicit does not buy us much. if (!MBP.SingleUseCondition) return false; MachineBasicBlock *NotNullSucc, *NullSucc; if (MBP.Predicate == MachineBranchPredicate::PRED_NE) { NotNullSucc = MBP.TrueDest; NullSucc = MBP.FalseDest; } else { NotNullSucc = MBP.FalseDest; NullSucc = MBP.TrueDest; } // We handle the simplest case for now. We can potentially do better by using // the machine dominator tree. if (NotNullSucc->pred_size() != 1) return false; // Starting with a code fragment like: // // test %RAX, %RAX // jne LblNotNull // // LblNull: // callq throw_NullPointerException // // LblNotNull: // Inst0 // Inst1 // ... // Def = Load (%RAX + <offset>) // ... // // // we want to end up with // // Def = TrappingLoad (%RAX + <offset>), LblNull // jmp LblNotNull ;; explicit or fallthrough // // LblNotNull: // Inst0 // Inst1 // ... // // LblNull: // callq throw_NullPointerException // unsigned PointerReg = MBP.LHS.getReg(); // As we scan NotNullSucc for a suitable load instruction, we keep track of // the registers defined and used by the instructions we scan past. This bit // of information lets us decide if it is legal to hoist the load instruction // we find (if we do find such an instruction) to before NotNullSucc. DenseSet<unsigned> RegDefs, RegUses; // Returns true if it is safe to reorder MI to before NotNullSucc. auto IsSafeToHoist = [&](MachineInstr *MI) { // Right now we don't want to worry about LLVM's memory model. This can be // made more precise later. for (auto *MMO : MI->memoperands()) if (!MMO->isUnordered()) return false; for (auto &MO : MI->operands()) { if (MO.isReg() && MO.getReg()) { for (unsigned Reg : RegDefs) if (TRI->regsOverlap(Reg, MO.getReg())) return false; // We found a write-after-write or read-after-write if (MO.isDef()) for (unsigned Reg : RegUses) if (TRI->regsOverlap(Reg, MO.getReg())) return false; // We found a write-after-read } } return true; }; for (auto MII = NotNullSucc->begin(), MIE = NotNullSucc->end(); MII != MIE; ++MII) { MachineInstr *MI = &*MII; unsigned BaseReg, Offset; if (TII->getMemOpBaseRegImmOfs(MI, BaseReg, Offset, TRI)) if (MI->mayLoad() && !MI->isPredicable() && BaseReg == PointerReg && Offset < PageSize && MI->getDesc().getNumDefs() <= 1 && IsSafeToHoist(MI)) { NullCheckList.emplace_back(MI, MBP.ConditionDef, &MBB, NotNullSucc, NullSucc); return true; } // MI did not match our criteria for conversion to a trapping load. Check // if we can continue looking. if (MI->mayStore() || MI->hasUnmodeledSideEffects()) return false; for (auto *MMO : MI->memoperands()) // Right now we don't want to worry about LLVM's memory model. if (!MMO->isUnordered()) return false; // It _may_ be okay to reorder a later load instruction across MI. Make a // note of its operands so that we can make the legality check if we find a // suitable load instruction: for (auto &MO : MI->operands()) { if (!MO.isReg() || !MO.getReg()) continue; if (MO.isDef()) RegDefs.insert(MO.getReg()); else RegUses.insert(MO.getReg()); } } return false; }