Example #1
0
  /// Hoists the necessary check for beginning and end of the induction
  /// encapsulated by this access function to the header.
  void hoistCheckToPreheader(ArraySemanticsCall CheckToHoist,
                             SILBasicBlock *Preheader,
                             DominanceInfo *DT) {
    ApplyInst *AI = CheckToHoist;
    SILLocation Loc = AI->getLoc();
    SILBuilderWithScope Builder(Preheader->getTerminator(), AI);

    // Get the first induction value.
    auto FirstVal = Ind->getFirstValue();
    // Clone the struct for the start index.
    auto Start = cast<SILInstruction>(CheckToHoist.getIndex())
                     ->clone(Preheader->getTerminator());
    // Set the new start index to the first value of the induction.
    Start->setOperand(0, FirstVal);

      // Clone and fixup the load, retain sequence to the header.
    auto NewCheck = CheckToHoist.copyTo(Preheader->getTerminator(), DT);
    NewCheck->setOperand(1, Start);

    // Get the last induction value.
    auto LastVal = Ind->getLastValue(Loc, Builder);
    // Clone the struct for the end index.
    auto End = cast<SILInstruction>(CheckToHoist.getIndex())
                   ->clone(Preheader->getTerminator());
    // Set the new end index to the last value of the induction.
    End->setOperand(0, LastVal);

    NewCheck = CheckToHoist.copyTo(Preheader->getTerminator(), DT);
    NewCheck->setOperand(1, End);
  }
Example #2
0
/// Returns true if the block \p BB is terminated with a cond_br based on an
/// availability check.
static bool isAvailabilityCheck(SILBasicBlock *BB) {
  CondBranchInst *CBR = dyn_cast<CondBranchInst>(BB->getTerminator());
  if (!CBR)
    return false;
  
  ApplyInst *AI = dyn_cast<ApplyInst>(CBR->getCondition());
  if (!AI)
    return false;
  
  SILFunction *F = AI->getCalleeFunction();
  if (!F || !F->hasSemanticsAttrs())
    return false;

  return F->hasSemanticsAttrsThatStartsWith("availability");
}
Example #3
0
/// Returns the Apply and WitnessMethod instructions that use the
/// open_existential_addr instructions, or null if at least one of the
/// instructions is missing.
static ApplyWitnessPair getOpenExistentialUsers(OpenExistentialAddrInst *OE) {
  ApplyInst *AI = nullptr;
  WitnessMethodInst *WMI = nullptr;
  ApplyWitnessPair Empty = std::make_pair(nullptr, nullptr);

  for (auto *UI : getNonDebugUses(OE)) {
    auto *User = UI->getUser();
    if (!isa<WitnessMethodInst>(User) &&
        User->isTypeDependentOperand(UI->getOperandNumber()))
      continue;
    // Check that we have a single Apply user.
    if (auto *AA = dyn_cast<ApplyInst>(User)) {
      if (AI)
        return Empty;

      AI = AA;
      continue;
    }

    // Check that we have a single WMI user.
    if (auto *W = dyn_cast<WitnessMethodInst>(User)) {
      if (WMI)
        return Empty;

      WMI = W;
      continue;
    }

    // Unknown instruction.
    return Empty;
  }

  // Both instructions need to exist.
  if (!WMI || !AI)
    return Empty;

  // Make sure that the WMI and AI match.
  if (AI->getCallee() != WMI)
    return Empty;

  // We have exactly the pattern that we expected.
  return std::make_pair(AI, WMI);
}
Example #4
0
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
                                     ArrayRef<ApplyInst *> Calls) {
  LLVM_DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
             << Demangle::demangleSymbolAsString(InitF->getName())
             << " : " << Calls.size() << "\n");
  // Map each initializer-containing function to its final initializer call.
  llvm::DenseMap<SILFunction *, ApplyInst *> ParentFuncs;
  for (auto *AI : Calls) {
    assert(AI->getNumArguments() == 0 && "ill-formed global init call");
    assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
           == InitF && "wrong init call");
    SILFunction *ParentF = AI->getFunction();
    DominanceInfo *DT = DA->get(ParentF);
    ApplyInst *HoistAI =
        getHoistedApplyForInitializer(AI, DT, InitF, ParentF, ParentFuncs);

    // If we were unable to find anything, just go onto the next apply.
    if (!HoistAI) {
      continue;
    }

    // Otherwise, move this call to the outermost loop preheader.
    SILBasicBlock *BB = HoistAI->getParent();
    typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
    DomTreeNode *Node = DT->getNode(BB);
    while (Node) {
      SILBasicBlock *DomParentBB = Node->getBlock();
      if (isAvailabilityCheck(DomParentBB)) {
        LLVM_DEBUG(llvm::dbgs() << "  don't hoist above availability check "
                                   "at bb"
                                << DomParentBB->getDebugID() << "\n");
        break;
      }
      BB = DomParentBB;
      if (!isInLoop(BB))
        break;
      Node = Node->getIDom();
    }

    if (BB == HoistAI->getParent()) {
      // BB is either unreachable or not in a loop.
      LLVM_DEBUG(llvm::dbgs() << "  skipping (not in a loop): " << *HoistAI
                              << "  in " << HoistAI->getFunction()->getName()
                              << "\n");
      continue;
    }

    LLVM_DEBUG(llvm::dbgs() << "  hoisting: " << *HoistAI << "  in "
                       << HoistAI->getFunction()->getName() << "\n");
    HoistAI->moveBefore(&*BB->begin());
    placeFuncRef(HoistAI, DT);
    HasChanged = true;
  }
}
Example #5
0
ApplyInst *SILGlobalOpt::getHoistedApplyForInitializer(
    ApplyInst *AI, DominanceInfo *DT, SILFunction *InitF, SILFunction *ParentF,
    llvm::DenseMap<SILFunction *, ApplyInst *> &ParentFuncs) {
  auto PFI = ParentFuncs.find(ParentF);
  if (PFI == ParentFuncs.end()) {
    ParentFuncs[ParentF] = AI;

    // It's the first time we found a call to InitF in this function, so we
    // try to hoist it out of any loop.
    return AI;
  }

  // Found a replacement for this init call. Ensure the replacement dominates
  // the original call site.
  ApplyInst *CommonAI = PFI->second;
  assert(
      cast<FunctionRefInst>(CommonAI->getCallee())->getReferencedFunction() ==
          InitF &&
      "ill-formed global init call");
  SILBasicBlock *DomBB =
      DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());

  // We must not move initializers around availability-checks.
  if (isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT))
    return nullptr;

  ApplyInst *Result = nullptr;
  if (DomBB != CommonAI->getParent()) {
    CommonAI->moveBefore(&*DomBB->begin());
    placeFuncRef(CommonAI, DT);

    // Try to hoist the existing AI again if we move it to another block,
    // e.g. from a loop exit into the loop.
    Result = CommonAI;
  }

  AI->replaceAllUsesWith(CommonAI);
  AI->eraseFromParent();
  HasChanged = true;
  return Result;
}
Example #6
0
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
                                     ArrayRef<ApplyInst*> Calls) {
  DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
        << demangle_wrappers::demangleSymbolAsString(InitF->getName())
        << " : " << Calls.size() << "\n");
  // Map each initializer-containing function to its final initializer call.
  llvm::DenseMap<SILFunction*, ApplyInst*> ParentFuncs;
  for (auto *AI : Calls) {
    assert(AI->getNumArguments() == 0 && "ill-formed global init call");
    assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
           == InitF && "wrong init call");

    SILFunction *ParentF = AI->getFunction();
    DominanceInfo *DT = DA->get(ParentF);
    auto PFI = ParentFuncs.find(ParentF);
    ApplyInst *HoistAI = nullptr;
    if (PFI != ParentFuncs.end()) {
      // Found a replacement for this init call.
      // Ensure the replacement dominates the original call site.
      ApplyInst *CommonAI = PFI->second;
      assert(cast<FunctionRefInst>(CommonAI->getCallee())
             ->getReferencedFunction() == InitF &&
             "ill-formed global init call");
      SILBasicBlock *DomBB =
        DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
      
      // We must not move initializers around availability-checks.
      if (!isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT)) {
        if (DomBB != CommonAI->getParent()) {
          CommonAI->moveBefore(&*DomBB->begin());
          placeFuncRef(CommonAI, DT);
          
          // Try to hoist the existing AI again if we move it to another block,
          // e.g. from a loop exit into the loop.
          HoistAI = CommonAI;
        }
        AI->replaceAllUsesWith(CommonAI);
        AI->eraseFromParent();
        HasChanged = true;
      }
    } else {
      ParentFuncs[ParentF] = AI;
      
      // It's the first time we found a call to InitF in this function, so we
      // try to hoist it out of any loop.
      HoistAI = AI;
    }
    if (HoistAI) {
      // Move this call to the outermost loop preheader.
      SILBasicBlock *BB = HoistAI->getParent();
      typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
      DomTreeNode *Node = DT->getNode(BB);
      while (Node) {
        SILBasicBlock *DomParentBB = Node->getBlock();
        if (isAvailabilityCheck(DomParentBB)) {
          DEBUG(llvm::dbgs() << "  don't hoist above availability check at bb" <<
                DomParentBB->getDebugID() << "\n");
          break;
        }
        BB = DomParentBB;
        if (!isInLoop(BB))
          break;
        Node = Node->getIDom();
      }
      if (BB == HoistAI->getParent()) {
        // BB is either unreachable or not in a loop.
        DEBUG(llvm::dbgs() << "  skipping (not in a loop): " << *HoistAI
              << "  in " << HoistAI->getFunction()->getName() << "\n");
      }
      else {
        DEBUG(llvm::dbgs() << "  hoisting: " << *HoistAI
              << "  in " << HoistAI->getFunction()->getName() << "\n");
        HoistAI->moveBefore(&*BB->begin());
        placeFuncRef(HoistAI, DT);
        HasChanged = true;
      }
    }
  }
}
Example #7
0
/// Try to CSE the users of \p From to the users of \p To.
/// The original users of \p To are passed in ToApplyWitnessUsers.
/// Returns true on success.
static bool tryToCSEOpenExtCall(OpenExistentialAddrInst *From,
                                OpenExistentialAddrInst *To,
                                ApplyWitnessPair ToApplyWitnessUsers,
                                DominanceInfo *DA) {
  assert(From != To && "Can't replace instruction with itself");

  ApplyInst *FromAI = nullptr;
  ApplyInst *ToAI = nullptr;
  WitnessMethodInst *FromWMI = nullptr;
  WitnessMethodInst *ToWMI = nullptr;
  std::tie(FromAI, FromWMI) = getOpenExistentialUsers(From);
  std::tie(ToAI, ToWMI) = ToApplyWitnessUsers;

  // Make sure that the OEA instruction has exactly two expected users.
  if (!FromAI || !ToAI || !FromWMI || !ToWMI)
    return false;

  // Make sure we are calling the same method.
  if (FromWMI->getMember() != ToWMI->getMember())
    return false;

  // We are going to reuse the TO-WMI, so make sure it dominates the call site.
  if (!DA->properlyDominates(ToWMI, FromWMI))
    return false;

  SILBuilder Builder(FromAI);
  // Make archetypes used by the ToAI available to the builder.
  SILOpenedArchetypesTracker OpenedArchetypesTracker(*FromAI->getFunction());
  OpenedArchetypesTracker.registerUsedOpenedArchetypes(ToAI);
  Builder.setOpenedArchetypesTracker(&OpenedArchetypesTracker);

  assert(FromAI->getArguments().size() == ToAI->getArguments().size() &&
         "Invalid number of arguments");

  // Don't handle any apply instructions that involve substitutions.
  if (ToAI->getSubstitutions().size() != 1) return false;

  // Prepare the Apply args.
  SmallVector<SILValue, 8> Args;
  for (auto Op : FromAI->getArguments()) {
      Args.push_back(Op == From ? To : Op);
  }

  auto FnTy = ToAI->getSubstCalleeSILType();
  auto ResTy = FnTy.castTo<SILFunctionType>()->getSILResult();

  ApplyInst *NAI = Builder.createApply(ToAI->getLoc(), ToWMI, FnTy, ResTy,
                                       ToAI->getSubstitutions(), Args,
                                       ToAI->isNonThrowing());
  FromAI->replaceAllUsesWith(NAI);
  FromAI->eraseFromParent();
  NumOpenExtRemoved++;
  return true;
}
Example #8
0
/// Simplify the following two frontend patterns:
///
///   %payload_addr = init_enum_data_addr %payload_allocation
///   store %payload to %payload_addr
///   inject_enum_addr %payload_allocation, $EnumType.case
///
///   inject_enum_add %nopayload_allocation, $EnumType.case
///
/// for a concrete enum type $EnumType.case to:
///
///   %1 = enum $EnumType, $EnumType.case, %payload
///   store %1 to %payload_addr
///
///   %1 = enum $EnumType, $EnumType.case
///   store %1 to %nopayload_addr
///
/// We leave the cleaning up to mem2reg.
SILInstruction *
SILCombiner::visitInjectEnumAddrInst(InjectEnumAddrInst *IEAI) {
  // Given an inject_enum_addr of a concrete type without payload, promote it to
  // a store of an enum. Mem2reg/load forwarding will clean things up for us. We
  // can't handle the payload case here due to the flow problems caused by the
  // dependency in between the enum and its data.

  assert(IEAI->getOperand()->getType().isAddress() && "Must be an address");
  Builder.setCurrentDebugScope(IEAI->getDebugScope());

  if (IEAI->getOperand()->getType().isAddressOnly(IEAI->getModule())) {
    // Check for the following pattern inside the current basic block:
    // inject_enum_addr %payload_allocation, $EnumType.case1
    // ... no insns storing anything into %payload_allocation
    // select_enum_addr  %payload_allocation,
    //                   case $EnumType.case1: %Result1,
    //                   case case $EnumType.case2: %bResult2
    //                   ...
    //
    // Replace the select_enum_addr by %Result1

    auto *Term = IEAI->getParent()->getTerminator();
    if (isa<CondBranchInst>(Term) || isa<SwitchValueInst>(Term)) {
      auto BeforeTerm = std::prev(std::prev(IEAI->getParent()->end()));
      auto *SEAI = dyn_cast<SelectEnumAddrInst>(BeforeTerm);
      if (!SEAI)
        return nullptr;

      if (SEAI->getOperand() != IEAI->getOperand())
        return nullptr;

      SILBasicBlock::iterator II = IEAI->getIterator();
      StoreInst *SI = nullptr;
      for (;;) {
        SILInstruction *CI = &*II;
        if (CI == SEAI)
          break;
        ++II;
        SI = dyn_cast<StoreInst>(CI);
        if (SI) {
          if (SI->getDest() == IEAI->getOperand())
            return nullptr;
        }
        // Allow all instructions in between, which don't have any dependency to
        // the store.
        if (AA->mayWriteToMemory(&*II, IEAI->getOperand()))
          return nullptr;
      }

      auto *InjectedEnumElement = IEAI->getElement();
      auto Result = SEAI->getCaseResult(InjectedEnumElement);

      // Replace select_enum_addr by the result
      replaceInstUsesWith(*SEAI, Result);
      return nullptr;
    }

    // Check for the following pattern inside the current basic block:
    // inject_enum_addr %payload_allocation, $EnumType.case1
    // ... no insns storing anything into %payload_allocation
    // switch_enum_addr  %payload_allocation,
    //                   case $EnumType.case1: %bbX,
    //                   case case $EnumType.case2: %bbY
    //                   ...
    //
    // Replace the switch_enum_addr by select_enum_addr, switch_value.
    if (auto *SEI = dyn_cast<SwitchEnumAddrInst>(Term)) {
      if (SEI->getOperand() != IEAI->getOperand())
        return nullptr;

      SILBasicBlock::iterator II = IEAI->getIterator();
      StoreInst *SI = nullptr;
      for (;;) {
        SILInstruction *CI = &*II;
        if (CI == SEI)
          break;
        ++II;
        SI = dyn_cast<StoreInst>(CI);
        if (SI) {
          if (SI->getDest() == IEAI->getOperand())
            return nullptr;
        }
        // Allow all instructions in between, which don't have any dependency to
        // the store.
        if (AA->mayWriteToMemory(&*II, IEAI->getOperand()))
          return nullptr;
      }

      // Replace switch_enum_addr by a branch instruction.
      SILBuilderWithScope B(SEI);
      SmallVector<std::pair<EnumElementDecl *, SILValue>, 8> CaseValues;
      SmallVector<std::pair<SILValue, SILBasicBlock *>, 8> CaseBBs;

      auto IntTy = SILType::getBuiltinIntegerType(32, B.getASTContext());

      for (int i = 0, e = SEI->getNumCases(); i < e; ++i) {
        auto Pair = SEI->getCase(i);
        auto *IL = B.createIntegerLiteral(SEI->getLoc(), IntTy, APInt(32, i, false));
        SILValue ILValue = SILValue(IL);
        CaseValues.push_back(std::make_pair(Pair.first, ILValue));
        CaseBBs.push_back(std::make_pair(ILValue, Pair.second));
      }

      SILValue DefaultValue;
      SILBasicBlock *DefaultBB = nullptr;

      if (SEI->hasDefault()) {
        auto *IL = B.createIntegerLiteral(
          SEI->getLoc(), IntTy,
          APInt(32, static_cast<uint64_t>(SEI->getNumCases()), false));
        DefaultValue = SILValue(IL);
        DefaultBB = SEI->getDefaultBB();
      }

      auto *SEAI = B.createSelectEnumAddr(SEI->getLoc(), SEI->getOperand(), IntTy, DefaultValue, CaseValues);

      B.createSwitchValue(SEI->getLoc(), SILValue(SEAI), DefaultBB, CaseBBs);

      return eraseInstFromFunction(*SEI);
    }

    return nullptr;
  }

  // If the enum does not have a payload create the enum/store since we don't
  // need to worry about payloads.
  if (!IEAI->getElement()->hasArgumentType()) {
    EnumInst *E =
      Builder.createEnum(IEAI->getLoc(), SILValue(), IEAI->getElement(),
                          IEAI->getOperand()->getType().getObjectType());
    Builder.createStore(IEAI->getLoc(), E, IEAI->getOperand(),
                        StoreOwnershipQualifier::Unqualified);
    return eraseInstFromFunction(*IEAI);
  }

  // Ok, we have a payload enum, make sure that we have a store previous to
  // us...
  SILValue ASO = IEAI->getOperand();
  if (!isa<AllocStackInst>(ASO)) {
    return nullptr;
  }
  InitEnumDataAddrInst *DataAddrInst = nullptr;
  InjectEnumAddrInst *EnumAddrIns = nullptr;
  llvm::SmallPtrSet<SILInstruction *, 32> WriteSet;
  for (auto UsersIt : ASO->getUses()) {
    SILInstruction *CurrUser = UsersIt->getUser();
    if (CurrUser->isDeallocatingStack()) {
      // we don't care about the dealloc stack instructions
      continue;
    }
    if (isDebugInst(CurrUser) || isa<LoadInst>(CurrUser)) {
      // These Instructions are a non-risky use we can ignore
      continue;
    }
    if (auto *CurrInst = dyn_cast<InitEnumDataAddrInst>(CurrUser)) {
      if (DataAddrInst) {
        return nullptr;
      }
      DataAddrInst = CurrInst;
      continue;
    }
    if (auto *CurrInst = dyn_cast<InjectEnumAddrInst>(CurrUser)) {
      if (EnumAddrIns) {
        return nullptr;
      }
      EnumAddrIns = CurrInst;
      continue;
    }
    if (isa<StoreInst>(CurrUser)) {
      // The only MayWrite Instruction we can safely handle
      WriteSet.insert(CurrUser);
      continue;
    }
    // It is too risky to continue if it is any other instruction.
    return nullptr;
  }

  if (!DataAddrInst || !EnumAddrIns) {
    return nullptr;
  }
  assert((EnumAddrIns == IEAI) &&
         "Found InitEnumDataAddrInst differs from IEAI");
  // Found the DataAddrInst to this enum payload. Check if it has only use.
  if (!hasOneNonDebugUse(DataAddrInst))
    return nullptr;

  StoreInst *SI = dyn_cast<StoreInst>(getSingleNonDebugUser(DataAddrInst));
  ApplyInst *AI = dyn_cast<ApplyInst>(getSingleNonDebugUser(DataAddrInst));
  if (!SI && !AI) {
    return nullptr;
  }

  // Make sure the enum pattern instructions are the only ones which write to
  // this location
  if (!WriteSet.empty()) {
    // Analyze the instructions (implicit dominator analysis)
    // If we find any of MayWriteSet, return nullptr
    SILBasicBlock *InitEnumBB = DataAddrInst->getParent();
    assert(InitEnumBB && "DataAddrInst is not in a valid Basic Block");
    llvm::SmallVector<SILInstruction *, 64> Worklist;
    Worklist.push_back(IEAI);
    llvm::SmallPtrSet<SILBasicBlock *, 16> Preds;
    Preds.insert(IEAI->getParent());
    while (!Worklist.empty()) {
      SILInstruction *CurrIns = Worklist.pop_back_val();
      SILBasicBlock *CurrBB = CurrIns->getParent();

      if (CurrBB->isEntry() && CurrBB != InitEnumBB) {
        // reached prologue without encountering the init bb
        return nullptr;
      }

      for (auto InsIt = ++CurrIns->getIterator().getReverse();
           InsIt != CurrBB->rend(); ++InsIt) {
        SILInstruction *Ins = &*InsIt;
        if (Ins == DataAddrInst) {
          // don't care about what comes before init enum in the basic block
          break;
        }
        if (WriteSet.count(Ins) != 0) {
          return nullptr;
        }
      }

      if (CurrBB == InitEnumBB) {
        continue;
      }

      // Go to predecessors and do all that again
      for (SILBasicBlock *Pred : CurrBB->getPredecessorBlocks()) {
        // If it's already in the set, then we've already queued and/or
        // processed the predecessors.
        if (Preds.insert(Pred).second) {
          Worklist.push_back(&*Pred->rbegin());
        }
      }
    }
  }

  if (SI) {
    assert((SI->getDest() == DataAddrInst) &&
           "Can't find StoreInst with DataAddrInst as its destination");
    // In that case, create the payload enum/store.
    EnumInst *E = Builder.createEnum(
        DataAddrInst->getLoc(), SI->getSrc(), DataAddrInst->getElement(),
        DataAddrInst->getOperand()->getType().getObjectType());
    Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(),
                        StoreOwnershipQualifier::Unqualified);
    // Cleanup.
    eraseInstFromFunction(*SI);
    eraseInstFromFunction(*DataAddrInst);
    return eraseInstFromFunction(*IEAI);
  }

  // Check whether we have an apply initializing the enum.
  //  %iedai = init_enum_data_addr %enum_addr
  //         = apply(%iedai,...)
  //  inject_enum_addr %enum_addr
  //
  // We can localize the store to an alloc_stack.
  // Allowing us to perform the same optimization as for the store.
  //
  //  %alloca = alloc_stack
  //            apply(%alloca,...)
  //  %load = load %alloca
  //  %1 = enum $EnumType, $EnumType.case, %load
  //  store %1 to %nopayload_addr
  //
  assert(AI && "Must have an apply");
  unsigned ArgIdx = 0;
  Operand *EnumInitOperand = nullptr;
  for (auto &Opd : AI->getArgumentOperands()) {
    // Found an apply that initializes the enum. We can optimize this by
    // localizing the initialization to an alloc_stack and loading from it.
    DataAddrInst = dyn_cast<InitEnumDataAddrInst>(Opd.get());
    if (DataAddrInst && DataAddrInst->getOperand() == IEAI->getOperand() &&
        ArgIdx < AI->getSubstCalleeType()->getNumIndirectResults()) {
      EnumInitOperand = &Opd;
      break;
    }
    ++ArgIdx;
  }

  if (!EnumInitOperand) {
    return nullptr;
  }

  // Localize the address access.
  Builder.setInsertionPoint(AI);
  auto *AllocStack = Builder.createAllocStack(DataAddrInst->getLoc(),
                                              EnumInitOperand->get()->getType());
  EnumInitOperand->set(AllocStack);
  Builder.setInsertionPoint(std::next(SILBasicBlock::iterator(AI)));
  SILValue Load(Builder.createLoad(DataAddrInst->getLoc(), AllocStack,
                                   LoadOwnershipQualifier::Unqualified));
  EnumInst *E = Builder.createEnum(
      DataAddrInst->getLoc(), Load, DataAddrInst->getElement(),
      DataAddrInst->getOperand()->getType().getObjectType());
  Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(),
                      StoreOwnershipQualifier::Unqualified);
  Builder.createDeallocStack(DataAddrInst->getLoc(), AllocStack);
  eraseInstFromFunction(*DataAddrInst);
  return eraseInstFromFunction(*IEAI);
}