/// This should be called in top-down order of each def that needs its uses /// rewrited. The order that we visit uses for a given def is irrelevant. void SILSSAUpdater::RewriteUse(Operand &Op) { // Replicate function_refs to their uses. SILGen can't build phi nodes for // them and it would not make much sense anyways. if (auto *FR = dyn_cast<FunctionRefInst>(Op.get())) { assert(areIdentical(getAvailVals(AV)) && "The function_refs need to have the same value"); SILInstruction *User = Op.getUser(); auto *NewFR = FR->clone(User); Op.set(SILValue(NewFR, Op.get().getResultNumber())); return; } else if (auto *IL = dyn_cast<IntegerLiteralInst>(Op.get())) if (areIdentical(getAvailVals(AV))) { // Some llvm intrinsics don't like phi nodes as their constant inputs (e.g // ctlz). SILInstruction *User = Op.getUser(); auto *NewIL = IL->clone(User); Op.set(SILValue(NewIL, Op.get().getResultNumber())); return; } // Again we need to be careful here, because ssa construction (with the // existing representation) can change the operand from under us. UseWrapper UW(&Op); SILInstruction *User = Op.getUser(); SILValue NewVal = GetValueInMiddleOfBlock(User->getParent()); assert(NewVal && "Need a valid value"); ((Operand *)UW)->set((SILValue)NewVal); }
void SILInstruction::replaceAllUsesWithUndef() { SILModule &Mod = getModule(); while (!use_empty()) { Operand *Op = *use_begin(); Op->set(SILUndef::get(Op->get()->getType(), Mod)); } }
void ValueBase::replaceAllUsesWith(ValueBase *RHS) { assert(this != RHS && "Cannot RAUW a value with itself"); assert(getNumTypes() == RHS->getNumTypes() && "An instruction and the value base that it is being replaced by " "must have the same number of types"); while (!use_empty()) { Operand *Op = *use_begin(); Op->set(SILValue(RHS, Op->get().getResultNumber())); } }
bool SILValueOwnershipChecker::gatherUsers( SmallVectorImpl<BranchPropagatedUser> &lifetimeEndingUsers, SmallVectorImpl<BranchPropagatedUser> &nonLifetimeEndingUsers, SmallVectorImpl<BranchPropagatedUser> &implicitRegularUsers) { // See if Value is guaranteed. If we are guaranteed and not forwarding, then // we need to look through subobject uses for more uses. Otherwise, if we are // forwarding, we do not create any lifetime ending users/non lifetime ending // users since we verify against our base. auto ownershipKind = value.getOwnershipKind(); bool isGuaranteed = ownershipKind == ValueOwnershipKind::Guaranteed; bool isOwned = ownershipKind == ValueOwnershipKind::Owned; if (isGuaranteed && isGuaranteedForwardingValue(value)) return true; // Then gather up our initial list of users. SmallVector<Operand *, 8> users; std::copy(value->use_begin(), value->use_end(), std::back_inserter(users)); auto addCondBranchToList = [](SmallVectorImpl<BranchPropagatedUser> &list, CondBranchInst *cbi, unsigned operandIndex) { if (cbi->isConditionOperandIndex(operandIndex)) { list.emplace_back(cbi); return; } bool isTrueOperand = cbi->isTrueOperandIndex(operandIndex); list.emplace_back(cbi, isTrueOperand ? CondBranchInst::TrueIdx : CondBranchInst::FalseIdx); }; bool foundError = false; while (!users.empty()) { Operand *op = users.pop_back_val(); SILInstruction *user = op->getUser(); // If this op is a type dependent operand, skip it. It is not interesting // from an ownership perspective. if (user->isTypeDependentOperand(*op)) continue; bool isGuaranteedSubValue = false; if (isGuaranteed && isGuaranteedForwardingInst(op->getUser())) { isGuaranteedSubValue = true; } auto opOwnershipKindMap = op->getOwnershipKindMap(isGuaranteedSubValue); // If our ownership kind doesn't match, track that we found an error, emit // an error message optionally and then continue. if (!opOwnershipKindMap.canAcceptKind(ownershipKind)) { foundError = true; // If we did not support /any/ ownership kind, it means that we found a // conflicting answer so the kind map that was returned is the empty // map. Put out a more specific error here. if (!opOwnershipKindMap.data.any()) { handleError([&]() { llvm::errs() << "Function: '" << user->getFunction()->getName() << "'\n" << "Ill-formed SIL! Unable to compute ownership kind " "map for user?!\n" << "For terminator users, check that successors have " "compatible ownership kinds.\n" << "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n' << "Conv: " << ownershipKind << "\n\n"; }); continue; } handleError([&]() { llvm::errs() << "Function: '" << user->getFunction()->getName() << "'\n" << "Have operand with incompatible ownership?!\n" << "Value: " << op->get() << "User: "******"Operand Number: " << op->getOperandNumber() << '\n' << "Conv: " << ownershipKind << '\n' << "OwnershipMap:\n" << opOwnershipKindMap << '\n'; }); continue; } auto lifetimeConstraint = opOwnershipKindMap.getLifetimeConstraint(ownershipKind); if (lifetimeConstraint == UseLifetimeConstraint::MustBeInvalidated) { LLVM_DEBUG(llvm::dbgs() << " Lifetime Ending User: "******" Regular User: "******"Our value is guaranteed and this is a forwarding instruction. " "Should have guaranteed ownership as well."); copy(result->getUses(), std::back_inserter(users)); } continue; } assert(user->getResults().empty()); auto *ti = dyn_cast<TermInst>(user); if (!ti) { continue; } // Otherwise if we have a terminator, add any as uses any end_borrow to // ensure that the subscope is completely enclsed within the super scope. We // require all of our arguments to be either trivial or guaranteed. for (auto &succ : ti->getSuccessors()) { auto *succBlock = succ.getBB(); // If we do not have any arguments, then continue. if (succBlock->args_empty()) continue; // Otherwise, make sure that all arguments are trivial or guaranteed. If // we fail, emit an error. // // TODO: We could ignore this error and emit a more specific error on the // actual terminator. for (auto *succArg : succBlock->getPhiArguments()) { // *NOTE* We do not emit an error here since we want to allow for more // specific errors to be found during use_verification. // // TODO: Add a flag that associates the terminator instruction with // needing to be verified. If it isn't verified appropriately, assert // when the verifier is destroyed. auto succArgOwnershipKind = succArg->getOwnershipKind(); if (!succArgOwnershipKind.isCompatibleWith(ownershipKind)) { // This is where the error would go. continue; } // If we have an any value, just continue. if (succArgOwnershipKind == ValueOwnershipKind::Any) continue; // Otherwise add all end_borrow users for this BBArg to the // implicit regular user list. We know that BBArg must be // completely joint post-dominated by these users, so we use // them to ensure that all of BBArg's uses are completely // enclosed within the end_borrow of this argument. for (auto *op : succArg->getUses()) { if (auto *ebi = dyn_cast<EndBorrowInst>(op->getUser())) { implicitRegularUsers.push_back(ebi); } } } } } // Return true if we did not have an error and false if we did find an error. // // The reason why we use this extra variable is to make sure that when we are // testing, we print out all mismatching pairs rather than just the first. return !foundError; }
void swift::visitAccessedAddress(SILInstruction *I, llvm::function_ref<void(Operand *)> visitor) { assert(I->mayReadOrWriteMemory()); // Reference counting instructions do not access user visible memory. if (isa<RefCountingInst>(I)) return; if (isa<DeallocationInst>(I)) return; if (auto apply = FullApplySite::isa(I)) { visitApplyAccesses(apply, visitor); return; } if (auto builtin = dyn_cast<BuiltinInst>(I)) { visitBuiltinAddress(builtin, visitor); return; } switch (I->getKind()) { default: I->dump(); llvm_unreachable("unexpected memory access."); case SILInstructionKind::AssignInst: visitor(&I->getAllOperands()[AssignInst::Dest]); return; case SILInstructionKind::CheckedCastAddrBranchInst: visitor(&I->getAllOperands()[CheckedCastAddrBranchInst::Src]); visitor(&I->getAllOperands()[CheckedCastAddrBranchInst::Dest]); return; case SILInstructionKind::CopyAddrInst: visitor(&I->getAllOperands()[CopyAddrInst::Src]); visitor(&I->getAllOperands()[CopyAddrInst::Dest]); return; #define NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Store##Name##Inst: #define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Store##Name##Inst: #include "swift/AST/ReferenceStorage.def" case SILInstructionKind::StoreInst: case SILInstructionKind::StoreBorrowInst: visitor(&I->getAllOperands()[StoreInst::Dest]); return; case SILInstructionKind::SelectEnumAddrInst: visitor(&I->getAllOperands()[0]); return; #define NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Load##Name##Inst: #define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Load##Name##Inst: #include "swift/AST/ReferenceStorage.def" case SILInstructionKind::InitExistentialAddrInst: case SILInstructionKind::InjectEnumAddrInst: case SILInstructionKind::LoadInst: case SILInstructionKind::LoadBorrowInst: case SILInstructionKind::OpenExistentialAddrInst: case SILInstructionKind::SwitchEnumAddrInst: case SILInstructionKind::UncheckedTakeEnumDataAddrInst: case SILInstructionKind::UnconditionalCheckedCastInst: { // Assuming all the above have only a single address operand. assert(I->getNumOperands() - I->getNumTypeDependentOperands() == 1); Operand *singleOperand = &I->getAllOperands()[0]; // Check the operand type because UnconditionalCheckedCastInst may operate // on a non-address. if (singleOperand->get()->getType().isAddress()) visitor(singleOperand); return; } // Non-access cases: these are marked with memory side effects, but, by // themselves, do not access formal memory. #define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \ case SILInstructionKind::Copy##Name##ValueInst: #include "swift/AST/ReferenceStorage.def" case SILInstructionKind::AbortApplyInst: case SILInstructionKind::AllocBoxInst: case SILInstructionKind::AllocExistentialBoxInst: case SILInstructionKind::AllocGlobalInst: case SILInstructionKind::BeginAccessInst: case SILInstructionKind::BeginApplyInst: case SILInstructionKind::BeginBorrowInst: case SILInstructionKind::BeginUnpairedAccessInst: case SILInstructionKind::BindMemoryInst: case SILInstructionKind::CheckedCastValueBranchInst: case SILInstructionKind::CondFailInst: case SILInstructionKind::CopyBlockInst: case SILInstructionKind::CopyBlockWithoutEscapingInst: case SILInstructionKind::CopyValueInst: case SILInstructionKind::DeinitExistentialAddrInst: case SILInstructionKind::DeinitExistentialValueInst: case SILInstructionKind::DestroyAddrInst: case SILInstructionKind::DestroyValueInst: case SILInstructionKind::EndAccessInst: case SILInstructionKind::EndApplyInst: case SILInstructionKind::EndBorrowInst: case SILInstructionKind::EndUnpairedAccessInst: case SILInstructionKind::EndLifetimeInst: case SILInstructionKind::ExistentialMetatypeInst: case SILInstructionKind::FixLifetimeInst: case SILInstructionKind::InitExistentialValueInst: case SILInstructionKind::IsUniqueInst: case SILInstructionKind::IsEscapingClosureInst: case SILInstructionKind::KeyPathInst: case SILInstructionKind::OpenExistentialBoxInst: case SILInstructionKind::OpenExistentialBoxValueInst: case SILInstructionKind::OpenExistentialValueInst: case SILInstructionKind::PartialApplyInst: case SILInstructionKind::ProjectValueBufferInst: case SILInstructionKind::YieldInst: case SILInstructionKind::UnwindInst: case SILInstructionKind::UncheckedOwnershipConversionInst: case SILInstructionKind::UncheckedRefCastAddrInst: case SILInstructionKind::UnconditionalCheckedCastAddrInst: case SILInstructionKind::UnconditionalCheckedCastValueInst: case SILInstructionKind::ValueMetatypeInst: return; } }
/// Simplify the following two frontend patterns: /// /// %payload_addr = init_enum_data_addr %payload_allocation /// store %payload to %payload_addr /// inject_enum_addr %payload_allocation, $EnumType.case /// /// inject_enum_add %nopayload_allocation, $EnumType.case /// /// for a concrete enum type $EnumType.case to: /// /// %1 = enum $EnumType, $EnumType.case, %payload /// store %1 to %payload_addr /// /// %1 = enum $EnumType, $EnumType.case /// store %1 to %nopayload_addr /// /// We leave the cleaning up to mem2reg. SILInstruction * SILCombiner::visitInjectEnumAddrInst(InjectEnumAddrInst *IEAI) { // Given an inject_enum_addr of a concrete type without payload, promote it to // a store of an enum. Mem2reg/load forwarding will clean things up for us. We // can't handle the payload case here due to the flow problems caused by the // dependency in between the enum and its data. assert(IEAI->getOperand()->getType().isAddress() && "Must be an address"); Builder.setCurrentDebugScope(IEAI->getDebugScope()); if (IEAI->getOperand()->getType().isAddressOnly(IEAI->getModule())) { // Check for the following pattern inside the current basic block: // inject_enum_addr %payload_allocation, $EnumType.case1 // ... no insns storing anything into %payload_allocation // select_enum_addr %payload_allocation, // case $EnumType.case1: %Result1, // case case $EnumType.case2: %bResult2 // ... // // Replace the select_enum_addr by %Result1 auto *Term = IEAI->getParent()->getTerminator(); if (isa<CondBranchInst>(Term) || isa<SwitchValueInst>(Term)) { auto BeforeTerm = std::prev(std::prev(IEAI->getParent()->end())); auto *SEAI = dyn_cast<SelectEnumAddrInst>(BeforeTerm); if (!SEAI) return nullptr; if (SEAI->getOperand() != IEAI->getOperand()) return nullptr; SILBasicBlock::iterator II = IEAI->getIterator(); StoreInst *SI = nullptr; for (;;) { SILInstruction *CI = &*II; if (CI == SEAI) break; ++II; SI = dyn_cast<StoreInst>(CI); if (SI) { if (SI->getDest() == IEAI->getOperand()) return nullptr; } // Allow all instructions in between, which don't have any dependency to // the store. if (AA->mayWriteToMemory(&*II, IEAI->getOperand())) return nullptr; } auto *InjectedEnumElement = IEAI->getElement(); auto Result = SEAI->getCaseResult(InjectedEnumElement); // Replace select_enum_addr by the result replaceInstUsesWith(*SEAI, Result); return nullptr; } // Check for the following pattern inside the current basic block: // inject_enum_addr %payload_allocation, $EnumType.case1 // ... no insns storing anything into %payload_allocation // switch_enum_addr %payload_allocation, // case $EnumType.case1: %bbX, // case case $EnumType.case2: %bbY // ... // // Replace the switch_enum_addr by select_enum_addr, switch_value. if (auto *SEI = dyn_cast<SwitchEnumAddrInst>(Term)) { if (SEI->getOperand() != IEAI->getOperand()) return nullptr; SILBasicBlock::iterator II = IEAI->getIterator(); StoreInst *SI = nullptr; for (;;) { SILInstruction *CI = &*II; if (CI == SEI) break; ++II; SI = dyn_cast<StoreInst>(CI); if (SI) { if (SI->getDest() == IEAI->getOperand()) return nullptr; } // Allow all instructions in between, which don't have any dependency to // the store. if (AA->mayWriteToMemory(&*II, IEAI->getOperand())) return nullptr; } // Replace switch_enum_addr by a branch instruction. SILBuilderWithScope B(SEI); SmallVector<std::pair<EnumElementDecl *, SILValue>, 8> CaseValues; SmallVector<std::pair<SILValue, SILBasicBlock *>, 8> CaseBBs; auto IntTy = SILType::getBuiltinIntegerType(32, B.getASTContext()); for (int i = 0, e = SEI->getNumCases(); i < e; ++i) { auto Pair = SEI->getCase(i); auto *IL = B.createIntegerLiteral(SEI->getLoc(), IntTy, APInt(32, i, false)); SILValue ILValue = SILValue(IL); CaseValues.push_back(std::make_pair(Pair.first, ILValue)); CaseBBs.push_back(std::make_pair(ILValue, Pair.second)); } SILValue DefaultValue; SILBasicBlock *DefaultBB = nullptr; if (SEI->hasDefault()) { auto *IL = B.createIntegerLiteral( SEI->getLoc(), IntTy, APInt(32, static_cast<uint64_t>(SEI->getNumCases()), false)); DefaultValue = SILValue(IL); DefaultBB = SEI->getDefaultBB(); } auto *SEAI = B.createSelectEnumAddr(SEI->getLoc(), SEI->getOperand(), IntTy, DefaultValue, CaseValues); B.createSwitchValue(SEI->getLoc(), SILValue(SEAI), DefaultBB, CaseBBs); return eraseInstFromFunction(*SEI); } return nullptr; } // If the enum does not have a payload create the enum/store since we don't // need to worry about payloads. if (!IEAI->getElement()->hasArgumentType()) { EnumInst *E = Builder.createEnum(IEAI->getLoc(), SILValue(), IEAI->getElement(), IEAI->getOperand()->getType().getObjectType()); Builder.createStore(IEAI->getLoc(), E, IEAI->getOperand(), StoreOwnershipQualifier::Unqualified); return eraseInstFromFunction(*IEAI); } // Ok, we have a payload enum, make sure that we have a store previous to // us... SILValue ASO = IEAI->getOperand(); if (!isa<AllocStackInst>(ASO)) { return nullptr; } InitEnumDataAddrInst *DataAddrInst = nullptr; InjectEnumAddrInst *EnumAddrIns = nullptr; llvm::SmallPtrSet<SILInstruction *, 32> WriteSet; for (auto UsersIt : ASO->getUses()) { SILInstruction *CurrUser = UsersIt->getUser(); if (CurrUser->isDeallocatingStack()) { // we don't care about the dealloc stack instructions continue; } if (isDebugInst(CurrUser) || isa<LoadInst>(CurrUser)) { // These Instructions are a non-risky use we can ignore continue; } if (auto *CurrInst = dyn_cast<InitEnumDataAddrInst>(CurrUser)) { if (DataAddrInst) { return nullptr; } DataAddrInst = CurrInst; continue; } if (auto *CurrInst = dyn_cast<InjectEnumAddrInst>(CurrUser)) { if (EnumAddrIns) { return nullptr; } EnumAddrIns = CurrInst; continue; } if (isa<StoreInst>(CurrUser)) { // The only MayWrite Instruction we can safely handle WriteSet.insert(CurrUser); continue; } // It is too risky to continue if it is any other instruction. return nullptr; } if (!DataAddrInst || !EnumAddrIns) { return nullptr; } assert((EnumAddrIns == IEAI) && "Found InitEnumDataAddrInst differs from IEAI"); // Found the DataAddrInst to this enum payload. Check if it has only use. if (!hasOneNonDebugUse(DataAddrInst)) return nullptr; StoreInst *SI = dyn_cast<StoreInst>(getSingleNonDebugUser(DataAddrInst)); ApplyInst *AI = dyn_cast<ApplyInst>(getSingleNonDebugUser(DataAddrInst)); if (!SI && !AI) { return nullptr; } // Make sure the enum pattern instructions are the only ones which write to // this location if (!WriteSet.empty()) { // Analyze the instructions (implicit dominator analysis) // If we find any of MayWriteSet, return nullptr SILBasicBlock *InitEnumBB = DataAddrInst->getParent(); assert(InitEnumBB && "DataAddrInst is not in a valid Basic Block"); llvm::SmallVector<SILInstruction *, 64> Worklist; Worklist.push_back(IEAI); llvm::SmallPtrSet<SILBasicBlock *, 16> Preds; Preds.insert(IEAI->getParent()); while (!Worklist.empty()) { SILInstruction *CurrIns = Worklist.pop_back_val(); SILBasicBlock *CurrBB = CurrIns->getParent(); if (CurrBB->isEntry() && CurrBB != InitEnumBB) { // reached prologue without encountering the init bb return nullptr; } for (auto InsIt = ++CurrIns->getIterator().getReverse(); InsIt != CurrBB->rend(); ++InsIt) { SILInstruction *Ins = &*InsIt; if (Ins == DataAddrInst) { // don't care about what comes before init enum in the basic block break; } if (WriteSet.count(Ins) != 0) { return nullptr; } } if (CurrBB == InitEnumBB) { continue; } // Go to predecessors and do all that again for (SILBasicBlock *Pred : CurrBB->getPredecessorBlocks()) { // If it's already in the set, then we've already queued and/or // processed the predecessors. if (Preds.insert(Pred).second) { Worklist.push_back(&*Pred->rbegin()); } } } } if (SI) { assert((SI->getDest() == DataAddrInst) && "Can't find StoreInst with DataAddrInst as its destination"); // In that case, create the payload enum/store. EnumInst *E = Builder.createEnum( DataAddrInst->getLoc(), SI->getSrc(), DataAddrInst->getElement(), DataAddrInst->getOperand()->getType().getObjectType()); Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(), StoreOwnershipQualifier::Unqualified); // Cleanup. eraseInstFromFunction(*SI); eraseInstFromFunction(*DataAddrInst); return eraseInstFromFunction(*IEAI); } // Check whether we have an apply initializing the enum. // %iedai = init_enum_data_addr %enum_addr // = apply(%iedai,...) // inject_enum_addr %enum_addr // // We can localize the store to an alloc_stack. // Allowing us to perform the same optimization as for the store. // // %alloca = alloc_stack // apply(%alloca,...) // %load = load %alloca // %1 = enum $EnumType, $EnumType.case, %load // store %1 to %nopayload_addr // assert(AI && "Must have an apply"); unsigned ArgIdx = 0; Operand *EnumInitOperand = nullptr; for (auto &Opd : AI->getArgumentOperands()) { // Found an apply that initializes the enum. We can optimize this by // localizing the initialization to an alloc_stack and loading from it. DataAddrInst = dyn_cast<InitEnumDataAddrInst>(Opd.get()); if (DataAddrInst && DataAddrInst->getOperand() == IEAI->getOperand() && ArgIdx < AI->getSubstCalleeType()->getNumIndirectResults()) { EnumInitOperand = &Opd; break; } ++ArgIdx; } if (!EnumInitOperand) { return nullptr; } // Localize the address access. Builder.setInsertionPoint(AI); auto *AllocStack = Builder.createAllocStack(DataAddrInst->getLoc(), EnumInitOperand->get()->getType()); EnumInitOperand->set(AllocStack); Builder.setInsertionPoint(std::next(SILBasicBlock::iterator(AI))); SILValue Load(Builder.createLoad(DataAddrInst->getLoc(), AllocStack, LoadOwnershipQualifier::Unqualified)); EnumInst *E = Builder.createEnum( DataAddrInst->getLoc(), Load, DataAddrInst->getElement(), DataAddrInst->getOperand()->getType().getObjectType()); Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(), StoreOwnershipQualifier::Unqualified); Builder.createDeallocStack(DataAddrInst->getLoc(), AllocStack); eraseInstFromFunction(*DataAddrInst); return eraseInstFromFunction(*IEAI); }