/// CreateEmptyPHI - Create a new PHI instruction in the specified block.
/// Reserve space for the operands but do not fill them in yet.
static Value *CreateEmptyPHI(BasicBlock *BB, unsigned NumPreds,
SSAUpdater *Updater) {
PHINode *PHI = PHINode::Create(Updater->PrototypeValue->getType(),
Updater->PrototypeValue->getName(),
&BB->front());
PHI->reserveOperandSpace(NumPreds);
return PHI;
}
/// ProcessInstruction - Given an instruction in the loop, check to see if it
/// has any uses that are outside the current loop. If so, insert LCSSA PHI
/// nodes and rewrite the uses.
bool LCSSA::ProcessInstruction(Instruction *Inst,
const SmallVectorImpl<BasicBlock*> &ExitBlocks) {
SmallVector<Use*, 16> UsesToRewrite;
BasicBlock *InstBB = Inst->getParent();
for (Value::use_iterator UI = Inst->use_begin(), E = Inst->use_end();
UI != E; ++UI) {
User *U = *UI;
BasicBlock *UserBB = cast<Instruction>(U)->getParent();
if (PHINode *PN = dyn_cast<PHINode>(U))
UserBB = PN->getIncomingBlock(UI);
if (InstBB != UserBB && !inLoop(UserBB))
UsesToRewrite.push_back(&UI.getUse());
}
// If there are no uses outside the loop, exit with no change.
if (UsesToRewrite.empty()) return false;
++NumLCSSA; // We are applying the transformation
// Invoke instructions are special in that their result value is not available
// along their unwind edge. The code below tests to see whether DomBB dominates
// the value, so adjust DomBB to the normal destination block, which is
// effectively where the value is first usable.
BasicBlock *DomBB = Inst->getParent();
if (InvokeInst *Inv = dyn_cast<InvokeInst>(Inst))
DomBB = Inv->getNormalDest();
DomTreeNode *DomNode = DT->getNode(DomBB);
SSAUpdater SSAUpdate;
SSAUpdate.Initialize(Inst->getType(), Inst->getName());
// Insert the LCSSA phi's into all of the exit blocks dominated by the
// value, and add them to the Phi's map.
for (SmallVectorImpl<BasicBlock*>::const_iterator BBI = ExitBlocks.begin(),
BBE = ExitBlocks.end(); BBI != BBE; ++BBI) {
BasicBlock *ExitBB = *BBI;
if (!DT->dominates(DomNode, DT->getNode(ExitBB))) continue;
// If we already inserted something for this BB, don't reprocess it.
if (SSAUpdate.HasValueForBlock(ExitBB)) continue;
PHINode *PN = PHINode::Create(Inst->getType(), Inst->getName()+".lcssa",
ExitBB->begin());
PN->reserveOperandSpace(PredCache.GetNumPreds(ExitBB));
// Add inputs from inside the loop for this PHI.
for (BasicBlock **PI = PredCache.GetPreds(ExitBB); *PI; ++PI) {
PN->addIncoming(Inst, *PI);
// If the exit block has a predecessor not within the loop, arrange for
// the incoming value use corresponding to that predecessor to be
// rewritten in terms of a different LCSSA PHI.
if (!inLoop(*PI))
UsesToRewrite.push_back(
&PN->getOperandUse(
PN->getOperandNumForIncomingValue(PN->getNumIncomingValues()-1)));
}
// Remember that this phi makes the value alive in this block.
SSAUpdate.AddAvailableValue(ExitBB, PN);
}
// Rewrite all uses outside the loop in terms of the new PHIs we just
// inserted.
for (unsigned i = 0, e = UsesToRewrite.size(); i != e; ++i) {
// If this use is in an exit block, rewrite to use the newly inserted PHI.
// This is required for correctness because SSAUpdate doesn't handle uses in
// the same block. It assumes the PHI we inserted is at the end of the
// block.
Instruction *User = cast<Instruction>(UsesToRewrite[i]->getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(*UsesToRewrite[i]);
if (isa<PHINode>(UserBB->begin()) &&
isExitBlock(UserBB, ExitBlocks)) {
UsesToRewrite[i]->set(UserBB->begin());
continue;
}
// Otherwise, do full PHI insertion.
SSAUpdate.RewriteUse(*UsesToRewrite[i]);
}
return true;
}
/// SimplifyStoreAtEndOfBlock - Turn things like:
/// if () { *P = v1; } else { *P = v2 }
/// into a phi node with a store in the successor.
///
/// Simplify things like:
/// *P = v1; if () { *P = v2; }
/// into a phi node with a store in the successor.
///
bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
BasicBlock *StoreBB = SI.getParent();
// Check to see if the successor block has exactly two incoming edges. If
// so, see if the other predecessor contains a store to the same location.
// if so, insert a PHI node (if needed) and move the stores down.
BasicBlock *DestBB = StoreBB->getTerminator()->getSuccessor(0);
// Determine whether Dest has exactly two predecessors and, if so, compute
// the other predecessor.
pred_iterator PI = pred_begin(DestBB);
BasicBlock *P = *PI;
BasicBlock *OtherBB = 0;
if (P != StoreBB)
OtherBB = P;
if (++PI == pred_end(DestBB))
return false;
P = *PI;
if (P != StoreBB) {
if (OtherBB)
return false;
OtherBB = P;
}
if (++PI != pred_end(DestBB))
return false;
// Bail out if all the relevant blocks aren't distinct (this can happen,
// for example, if SI is in an infinite loop)
if (StoreBB == DestBB || OtherBB == DestBB)
return false;
// Verify that the other block ends in a branch and is not otherwise empty.
BasicBlock::iterator BBI = OtherBB->getTerminator();
BranchInst *OtherBr = dyn_cast<BranchInst>(BBI);
if (!OtherBr || BBI == OtherBB->begin())
return false;
// If the other block ends in an unconditional branch, check for the 'if then
// else' case. there is an instruction before the branch.
StoreInst *OtherStore = 0;
if (OtherBr->isUnconditional()) {
--BBI;
// Skip over debugging info.
while (isa<DbgInfoIntrinsic>(BBI) ||
(isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy())) {
if (BBI==OtherBB->begin())
return false;
--BBI;
}
// If this isn't a store, isn't a store to the same location, or if the
// alignments differ, bail out.
OtherStore = dyn_cast<StoreInst>(BBI);
if (!OtherStore || OtherStore->getOperand(1) != SI.getOperand(1) ||
OtherStore->getAlignment() != SI.getAlignment())
return false;
} else {
// Otherwise, the other block ended with a conditional branch. If one of the
// destinations is StoreBB, then we have the if/then case.
if (OtherBr->getSuccessor(0) != StoreBB &&
OtherBr->getSuccessor(1) != StoreBB)
return false;
// Okay, we know that OtherBr now goes to Dest and StoreBB, so this is an
// if/then triangle. See if there is a store to the same ptr as SI that
// lives in OtherBB.
for (;; --BBI) {
// Check to see if we find the matching store.
if ((OtherStore = dyn_cast<StoreInst>(BBI))) {
if (OtherStore->getOperand(1) != SI.getOperand(1) ||
OtherStore->getAlignment() != SI.getAlignment())
return false;
break;
}
// If we find something that may be using or overwriting the stored
// value, or if we run out of instructions, we can't do the xform.
if (BBI->mayReadFromMemory() || BBI->mayWriteToMemory() ||
BBI == OtherBB->begin())
return false;
}
// In order to eliminate the store in OtherBr, we have to
// make sure nothing reads or overwrites the stored value in
// StoreBB.
for (BasicBlock::iterator I = StoreBB->begin(); &*I != &SI; ++I) {
// FIXME: This should really be AA driven.
if (I->mayReadFromMemory() || I->mayWriteToMemory())
return false;
}
}
// Insert a PHI node now if we need it.
Value *MergedVal = OtherStore->getOperand(0);
if (MergedVal != SI.getOperand(0)) {
PHINode *PN = PHINode::Create(MergedVal->getType(), "storemerge");
PN->reserveOperandSpace(2);
PN->addIncoming(SI.getOperand(0), SI.getParent());
PN->addIncoming(OtherStore->getOperand(0), OtherBB);
MergedVal = InsertNewInstBefore(PN, DestBB->front());
}
// Advance to a place where it is safe to insert the new store and
// insert it.
BBI = DestBB->getFirstNonPHI();
InsertNewInstBefore(new StoreInst(MergedVal, SI.getOperand(1),
OtherStore->isVolatile(),
SI.getAlignment()), *BBI);
// Nuke the old stores.
EraseInstFromFunction(SI);
EraseInstFromFunction(*OtherStore);
return true;
}
/// GetValueInMiddleOfBlock - Construct SSA form, materializing a value that
/// is live in the middle of the specified block.
///
/// GetValueInMiddleOfBlock is the same as GetValueAtEndOfBlock except in one
/// important case: if there is a definition of the rewritten value after the
/// 'use' in BB. Consider code like this:
///
/// X1 = ...
/// SomeBB:
/// use(X)
/// X2 = ...
/// br Cond, SomeBB, OutBB
///
/// In this case, there are two values (X1 and X2) added to the AvailableVals
/// set by the client of the rewriter, and those values are both live out of
/// their respective blocks. However, the use of X happens in the *middle* of
/// a block. Because of this, we need to insert a new PHI node in SomeBB to
/// merge the appropriate values, and this value isn't live out of the block.
///
Value *SSAUpdater::GetValueInMiddleOfBlock(BasicBlock *BB) {
// If there is no definition of the renamed variable in this block, just use
// GetValueAtEndOfBlock to do our work.
if (!HasValueForBlock(BB))
return GetValueAtEndOfBlock(BB);
// Otherwise, we have the hard case. Get the live-in values for each
// predecessor.
SmallVector<std::pair<BasicBlock*, Value*>, 8> PredValues;
Value *SingularValue = 0;
// We can get our predecessor info by walking the pred_iterator list, but it
// is relatively slow. If we already have PHI nodes in this block, walk one
// of them to get the predecessor list instead.
if (PHINode *SomePhi = dyn_cast<PHINode>(BB->begin())) {
for (unsigned i = 0, e = SomePhi->getNumIncomingValues(); i != e; ++i) {
BasicBlock *PredBB = SomePhi->getIncomingBlock(i);
Value *PredVal = GetValueAtEndOfBlock(PredBB);
PredValues.push_back(std::make_pair(PredBB, PredVal));
// Compute SingularValue.
if (i == 0)
SingularValue = PredVal;
else if (PredVal != SingularValue)
SingularValue = 0;
}
} else {
bool isFirstPred = true;
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
BasicBlock *PredBB = *PI;
Value *PredVal = GetValueAtEndOfBlock(PredBB);
PredValues.push_back(std::make_pair(PredBB, PredVal));
// Compute SingularValue.
if (isFirstPred) {
SingularValue = PredVal;
isFirstPred = false;
} else if (PredVal != SingularValue)
SingularValue = 0;
}
}
// If there are no predecessors, just return undef.
if (PredValues.empty())
return UndefValue::get(PrototypeValue->getType());
// Otherwise, if all the merged values are the same, just use it.
if (SingularValue != 0)
return SingularValue;
// Otherwise, we do need a PHI: check to see if we already have one available
// in this block that produces the right value.
if (isa<PHINode>(BB->begin())) {
DenseMap<BasicBlock*, Value*> ValueMapping(PredValues.begin(),
PredValues.end());
PHINode *SomePHI;
for (BasicBlock::iterator It = BB->begin();
(SomePHI = dyn_cast<PHINode>(It)); ++It) {
if (IsEquivalentPHI(SomePHI, ValueMapping))
return SomePHI;
}
}
// Ok, we have no way out, insert a new one now.
PHINode *InsertedPHI = PHINode::Create(PrototypeValue->getType(),
PrototypeValue->getName(),
&BB->front());
InsertedPHI->reserveOperandSpace(PredValues.size());
// Fill in all the predecessors of the PHI.
for (unsigned i = 0, e = PredValues.size(); i != e; ++i)
InsertedPHI->addIncoming(PredValues[i].second, PredValues[i].first);
// See if the PHI node can be merged to a single value. This can happen in
// loop cases when we get a PHI of itself and one other value.
if (Value *ConstVal = InsertedPHI->hasConstantValue()) {
InsertedPHI->eraseFromParent();
return ConstVal;
}
// If the client wants to know about all new instructions, tell it.
if (InsertedPHIs) InsertedPHIs->push_back(InsertedPHI);
DEBUG(dbgs() << " Inserted PHI: " << *InsertedPHI << "\n");
return InsertedPHI;
}
/// InsertUniqueBackedgeBlock - This method is called when the specified loop
/// has more than one backedge in it. If this occurs, revector all of these
/// backedges to target a new basic block and have that block branch to the loop
/// header. This ensures that loops have exactly one backedge.
///
BasicBlock *
LoopSimplify::InsertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader) {
assert(L->getNumBackEdges() > 1 && "Must have > 1 backedge!");
// Get information about the loop
BasicBlock *Header = L->getHeader();
Function *F = Header->getParent();
// Unique backedge insertion currently depends on having a preheader.
if (!Preheader)
return 0;
// Figure out which basic blocks contain back-edges to the loop header.
std::vector<BasicBlock*> BackedgeBlocks;
for (pred_iterator I = pred_begin(Header), E = pred_end(Header); I != E; ++I){
BasicBlock *P = *I;
// Indirectbr edges cannot be split, so we must fail if we find one.
if (isa<IndirectBrInst>(P->getTerminator()))
return 0;
if (P != Preheader) BackedgeBlocks.push_back(P);
}
// Create and insert the new backedge block...
BasicBlock *BEBlock = BasicBlock::Create(Header->getContext(),
Header->getName()+".backedge", F);
BranchInst *BETerminator = BranchInst::Create(Header, BEBlock);
DEBUG(dbgs() << "LoopSimplify: Inserting unique backedge block "
<< BEBlock->getName() << "\n");
// Move the new backedge block to right after the last backedge block.
Function::iterator InsertPos = BackedgeBlocks.back(); ++InsertPos;
F->getBasicBlockList().splice(InsertPos, F->getBasicBlockList(), BEBlock);
// Now that the block has been inserted into the function, create PHI nodes in
// the backedge block which correspond to any PHI nodes in the header block.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *PN = cast<PHINode>(I);
PHINode *NewPN = PHINode::Create(PN->getType(), PN->getName()+".be",
BETerminator);
NewPN->reserveOperandSpace(BackedgeBlocks.size());
if (AA) AA->copyValue(PN, NewPN);
// Loop over the PHI node, moving all entries except the one for the
// preheader over to the new PHI node.
unsigned PreheaderIdx = ~0U;
bool HasUniqueIncomingValue = true;
Value *UniqueValue = 0;
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
BasicBlock *IBB = PN->getIncomingBlock(i);
Value *IV = PN->getIncomingValue(i);
if (IBB == Preheader) {
PreheaderIdx = i;
} else {
NewPN->addIncoming(IV, IBB);
if (HasUniqueIncomingValue) {
if (UniqueValue == 0)
UniqueValue = IV;
else if (UniqueValue != IV)
HasUniqueIncomingValue = false;
}
}
}
// Delete all of the incoming values from the old PN except the preheader's
assert(PreheaderIdx != ~0U && "PHI has no preheader entry??");
if (PreheaderIdx != 0) {
PN->setIncomingValue(0, PN->getIncomingValue(PreheaderIdx));
PN->setIncomingBlock(0, PN->getIncomingBlock(PreheaderIdx));
}
// Nuke all entries except the zero'th.
for (unsigned i = 0, e = PN->getNumIncomingValues()-1; i != e; ++i)
PN->removeIncomingValue(e-i, false);
// Finally, add the newly constructed PHI node as the entry for the BEBlock.
PN->addIncoming(NewPN, BEBlock);
// As an optimization, if all incoming values in the new PhiNode (which is a
// subset of the incoming values of the old PHI node) have the same value,
// eliminate the PHI Node.
if (HasUniqueIncomingValue) {
NewPN->replaceAllUsesWith(UniqueValue);
if (AA) AA->deleteValue(NewPN);
BEBlock->getInstList().erase(NewPN);
}
}
// Now that all of the PHI nodes have been inserted and adjusted, modify the
// backedge blocks to just to the BEBlock instead of the header.
for (unsigned i = 0, e = BackedgeBlocks.size(); i != e; ++i) {
TerminatorInst *TI = BackedgeBlocks[i]->getTerminator();
for (unsigned Op = 0, e = TI->getNumSuccessors(); Op != e; ++Op)
if (TI->getSuccessor(Op) == Header)
TI->setSuccessor(Op, BEBlock);
}
//===--- Update all analyses which we must preserve now -----------------===//
// Update Loop Information - we know that this block is now in the current
// loop and all parent loops.
L->addBasicBlockToLoop(BEBlock, LI->getBase());
// Update dominator information
DT->splitBlock(BEBlock);
return BEBlock;
}
bool ModuloSchedulerDriverPass::runOnLoop(Loop *IncomingLoop, LPPassManager &LPM_Ref) {
subscripts subs(IncomingLoop);
if (!loop_is_ms_able(IncomingLoop) ) return false;
// The header before the parallelized loop will be placed here
BasicBlock* preheader = IncomingLoop->getLoopPreheader();
assert(preheader && "Unable to get a hold of the preheader");
// Balance all BasicBlocks in this loop
for (Loop::block_iterator it=IncomingLoop->block_begin(); it!=IncomingLoop->block_end();++it) {
duplicateValuesWithMultipleUses(*it,subs.getInductionVar());
}
// For each BB in loop
for (Loop::block_iterator it=IncomingLoop->block_begin(); it!=IncomingLoop->block_end();++it) {
instructionPriority ip(*it);
(*it)->setName("PipelinedLoop");
// ++++++++ Preheader part +++++++++
// Make a copy of the body for each instruction. Place a pointer to the
// parallel cloned instruction in the map below. Later on we will replace it
// with a PHINode.
DenseMap<const Value *, Value *> InstToPreheader;
// For each Instruction in body of the loop, clone, store, etc.
for (BasicBlock::iterator ib = (*it)->begin(), eb = (*it)->end(); ib!=eb; ++ib) {
// If this is NOT a phi node
if (!dyn_cast<PHINode>(ib)) {
// Get the priority of the instruction
unsigned int p = ip.getPriority(ib);
// This is the header version of each variable that goes into a PHI node.
// The other edge needs to come from the 'prev' iteration
// We subtract -1 because this is one iteration before
// Store the result into the map of the cloned
InstToPreheader[ib] = copyLoopBodyToHeader(ib, subs.getInductionVar(), preheader, p-1);
}
}
// ++++++++ Loop body part +++++++++
// For each of the cloned increment the indexs if needed and place the PHINode.
for (BasicBlock::iterator ib = (*it)->begin(), eb = (*it)->end(); ib!=eb; ++ib) {
// If this is NOT a phi node
if (!dyn_cast<PHINode>(ib)) {
unsigned int p = ip.getPriority(ib);
// If this variable is not dependent on i (not i:=i+1)
// then we need to replace each i to i+5 ...
// We also do not need to create a PHI node, etc.
if (!subs.isUsedByInductionVariable(ib)) {
incrementInductionVarIfUsed(ib,subs.getInductionVar(),p);
// Create the new PHI Node to replace the node
if (!dyn_cast<StoreInst>(ib) && !ib->isTerminator()) {
std::string newname = "glue" + (*it)->getName();
//PHINode* np = PHINode::Create(ib->getType(), "glue", *it);
PHINode* np = PHINode::Create(ib->getType(), newname, *it);
ib->replaceAllUsesWith(np);
np->reserveOperandSpace(2);
np->addIncoming(InstToPreheader[ib], preheader);
np->addIncoming(ib, *it);
np->moveBefore((*it)->begin());
}
}// end of if this is not an IV node (i:=i+1)
}
}
}
eliminateDuplicatedLoads(preheader);
for (Loop::block_iterator it=IncomingLoop->block_begin(); it!=IncomingLoop->block_end();++it) {
eliminateDuplicatedLoads(*it);
for (BasicBlock::iterator in = (*it)->begin(); in != (*it)->end(); ++in) {
foldAddInstructions(in);
}
}
return true;
}
