/*
* Sink IncRefs consumed off trace.
* Assumptions: Flow graph must not have critical edges, and the instructions
* have been annotated already by the DCE algorithm. This pass uses
* the REFCOUNT_CONSUMED* flags to copy IncRefs from the main trace to each
* exit trace that consumes the incremented pointer.
* 1. toSink = {}
* 2. iterate forwards over the main trace:
* * when a movable IncRef is found, insert into toSink list and mark
* it as DEAD.
* * If a decref of a dead incref is found, remove the corresponding
* incref from toSink, and mark the decref DEAD because too.
* * the first time we see a branch to an exit trace, process the
* exit tace.
* 3. to process an exit trace:
* * clone each IncRef found in toSink then prepend to the exit trace.
* * replace each use of the original incref's result with the new
* incref's result.
*/
void sinkIncRefs(Trace* trace, IRFactory* irFactory, DceState& state) {
assert(trace->isMain());
auto copyPropTrace = [] (Trace* trace) {
forEachInst(trace, copyProp);
};
WorkList toSink;
auto processExit = [&] (Trace* exit) {
// Sink REFCOUNT_CONSUMED_OFF_TRACE IncRefs before the first non-label
// instruction, and create a mapping between the original tmps to the sunk
// tmps so that we can later replace the original ones with the sunk ones.
std::vector<SSATmp*> sunkTmps(irFactory->numTmps(), nullptr);
for (auto* inst : boost::adaptors::reverse(toSink)) {
// prepend inserts an instruction to the beginning of a block, after
// the label. Therefore, we iterate through toSink in the reversed order.
IRInstruction* sunkInst = irFactory->gen(IncRef, inst->getSrc(0));
state[sunkInst].setLive();
exit->front()->prepend(sunkInst);
auto dstId = inst->getDst()->getId();
assert(!sunkTmps[dstId]);
sunkTmps[dstId] = sunkInst->getDst();
}
forEachInst(exit, [&](IRInstruction* inst) {
// Replace the original tmps with the sunk tmps.
for (uint32_t i = 0; i < inst->getNumSrcs(); ++i) {
SSATmp* src = inst->getSrc(i);
if (SSATmp* sunkTmp = sunkTmps[src->getId()]) {
inst->setSrc(i, sunkTmp);
}
}
});
// Do copyProp at last, because we need to keep REFCOUNT_CONSUMED_OFF_TRACE
// Movs as the prototypes for sunk instructions.
copyPropTrace(exit);
};
// An exit trace may be entered from multiple exit points. We keep track of
// which exit traces we already pushed sunk IncRefs to, so that we won't push
// them multiple times.
boost::dynamic_bitset<> pushedTo(irFactory->numBlocks());
forEachInst(trace, [&](IRInstruction* inst) {
if (inst->getOpcode() == IncRef) {
// Must be REFCOUNT_CONSUMED or REFCOUNT_CONSUMED_OFF_TRACE;
// otherwise, it should be already removed in optimizeRefCount.
if (state[inst].countConsumedOffTrace()) {
inst->setOpcode(Mov);
// Mark them as dead so that they'll be removed later.
state[inst].setDead();
// Put all REFCOUNT_CONSUMED_OFF_TRACE IncRefs to the sinking list.
toSink.push_back(inst);
} else if (!state[inst].isDead()) {
assert(state[inst].countConsumed());
}
}
if (inst->getOpcode() == DecRefNZ) {
IRInstruction* srcInst = inst->getSrc(0)->getInstruction();
if (state[srcInst].isDead()) {
state[inst].setDead();
// This may take O(I) time where I is the number of IncRefs
// in the main trace.
toSink.remove(srcInst);
}
}
if (Block* target = inst->getTaken()) {
if (!pushedTo[target->getId()]) {
pushedTo[target->getId()] = 1;
Trace* exit = target->getTrace();
if (exit != trace) processExit(exit);
}
}
});
// Do copyProp at last, because we need to keep REFCOUNT_CONSUMED_OFF_TRACE
// Movs as the prototypes for sunk instructions.
copyPropTrace(trace);
}