void StackInfo::registerInsts(IRManager& irm)
{
MethodDesc& md = irm.getMethodDesc();
if (!md.isStatic()) {
#ifdef _EM64T_
if ((md.isSynchronized() || md.isParentClassIsLikelyExceptionType())) {
EntryPointPseudoInst * entryPointInst = irm.getEntryPointInst();
offsetOfThis = (U_32)entryPointInst->thisOpnd->getMemOpndSubOpnd(MemOpndSubOpndKind_Displacement)->getImmValue();
} else {
offsetOfThis = 0;
}
#else
EntryPointPseudoInst * entryPointInst = irm.getEntryPointInst();
offsetOfThis = (U_32)entryPointInst->getOpnd(0)->getMemOpndSubOpnd(MemOpndSubOpndKind_Displacement)->getImmValue();
#endif
}
const Nodes& nodes = irm.getFlowGraph()->getNodes();
for (Nodes::const_iterator it = nodes.begin(), end = nodes.end(); it!=end; ++it) {
Node* node = *it;
if (node->isBlockNode()){
for (Inst * inst=(Inst*)node->getFirstInst(); inst!=NULL; inst=inst->getNextInst()){
if(inst->getMnemonic() == Mnemonic_CALL) {
(*stackDepthInfo)[(POINTER_SIZE_INT)inst->getCodeStartAddr()+inst->getCodeSize()]=
StackDepthInfo(((CallInst *)inst)->getCallingConventionClient().getCallingConvention()->getCalleeSavedRegs(OpndKind_GPReg).getMask(),
inst->getStackDepth(),
inst->getCodeSize());
}
}
}
}
hashTableSize = (U_32)stackDepthInfo->size();
}
void OSRPass::_run(IRManager & irm) {
splitCriticalEdges(irm);
computeDominatorsAndLoops(irm);
LoopTree* loopTree = irm.getLoopTree();
DominatorTree* dominatorTree = irm.getDominatorTree();
OSR osr(irm, irm.getMemoryManager(), loopTree, dominatorTree);
osr.runPass();
}
void TranslatorSession::postTranslatorCleanup() {
IRManager* irm = getCompilationContext()->getHIRManager();
ControlFlowGraph& flowGraph = irm->getFlowGraph();
MethodDesc& methodDesc = irm->getMethodDesc();
if (Log::isEnabled()) {
Log::out() << "PRINTING LOG: After Translator" << std::endl;
FlowGraph::printHIR(Log::out(), flowGraph, methodDesc);
}
FlowGraph::doTranslatorCleanupPhase(*irm);
if (Log::isEnabled()) {
Log::out() << "PRINTING LOG: After Cleanup" << std::endl;
FlowGraph::printHIR(Log::out(), flowGraph, methodDesc);
}
}
// this is the regular routine to be used to generate IR for a method
void TranslatorSession::translate() {
CompilationContext* cc = getCompilationContext();
IRManager* irm = cc->getHIRManager();
assert(irm);
irm->getTypeManager().setLazyResolutionMode(flags.lazyResolution);
MethodDesc& methodDesc = irm->getMethodDesc();
//create IRBuilder
MemoryManager& mm = cc->getCompilationLevelMemoryManager();
TranslatorAction* myAction = (TranslatorAction*)getAction();
IRBuilder* irb = (IRBuilder*)myAction->getIRBuilderAction()->createSession(mm);
irb->setCompilationContext(cc);
MemoryManager tmpMM("IRBuilder::tmpMM");
irb->init(irm, &flags, tmpMM);
JavaTranslator::translateMethod(*cc->getVMCompilationInterface(), methodDesc, *irb);
}
void findLoopsToUnroll(MemoryManager& tmpMM, IRManager& irm, UnrollInfos& result, const UnrollFlags& flags) {
ControlFlowGraph& fg = irm.getFlowGraph();
LoopTree* lt = fg.getLoopTree();
//find all loop exits
Edges loopExits(tmpMM);
const Nodes& nodes = fg.getNodes();
for (Nodes::const_iterator it = nodes.begin(), end = nodes.end(); it!=end; ++it) {
Node* node = *it;
LoopNode* loopNode = lt->getLoopNode(node, false);
if (loopNode == NULL) {
continue; //node not in a loop
}
if (!flags.unrollParentLoops && loopNode->getChild()!=NULL) {
continue; //skip parent loops
}
const Edges& edges = node->getOutEdges();
for (Edges::const_iterator ite = edges.begin(), ende = edges.end(); ite!=ende; ++ite) {
Edge* edge = *ite;
if (lt->isLoopExit(edge)) {
loopExits.push_back(edge);
}
}
}
//filter out all edges except branches
for (Edges::iterator ite = loopExits.begin(), ende = loopExits.end(); ite!=ende; ++ite) {
Edge* edge = *ite;
if (edge->isDispatchEdge() || edge->isUnconditionalEdge() || edge->isCatchEdge()) {
*ite = NULL;
continue;
}
Inst* lastInst = (Inst*)edge->getSourceNode()->getLastInst();
if (lastInst->isSwitch()) {
*ite = NULL;
continue;
}
assert(lastInst->isBranch());
assert(edge->isFalseEdge() || edge->isTrueEdge());
}
loopExits.erase(std::remove(loopExits.begin(), loopExits.end(), (Edge*)NULL), loopExits.end());
// analyze every loop exit and prepare unroll info
for (Edges::const_iterator ite = loopExits.begin(), ende = loopExits.end(); ite!=ende; ++ite) {
Edge* edge = *ite;
Node* sourceNode = edge->getSourceNode();
Inst* lastInst = (Inst*)sourceNode->getLastInst();
assert(lastInst->isBranch());
LoopUnrollInfo* info = prepareUnrollInfo(tmpMM, lt, lastInst->asBranchInst());
if (info == NULL) {
continue;
}
if (Log::isEnabled()) {
info->print(Log::out());
Log::out()<<std::endl;
}
result.push_back(info);
}
}
void
FixupVarsPass::_run(IRManager& irm) {
OptPass::computeDominators(irm);
DominatorTree* dominatorTree = irm.getDominatorTree();
ControlFlowGraph& flowGraph = irm.getFlowGraph();
DomFrontier frontier(irm.getNestedMemoryManager(),*dominatorTree,&flowGraph);
SSABuilder ssaBuilder(irm.getOpndManager(),irm.getInstFactory(),frontier,&flowGraph, irm.getOptimizerFlags());
bool phiInserted = ssaBuilder.fixupVars(&irm.getFlowGraph(), irm.getMethodDesc());
irm.setInSsa(true);
if (phiInserted)
irm.setSsaUpdated();
}
void
PersistentInstructionIdGenerator::runPass(IRManager& irm) {
MemoryManager mm("PersistentInstructionIdGenerator::runPass");
MethodDesc& methodDesc = irm.getMethodDesc();
StlVector<Node*> nodes(mm);
irm.getFlowGraph().getNodesPostOrder(nodes);
StlVector<Node*>::reverse_iterator i;
for(i = nodes.rbegin(); i != nodes.rend(); ++i) {
Node* node = *i;
Inst* label = (Inst*)node->getFirstInst();
for(Inst* inst = label->getNextInst(); inst != NULL; inst = inst->getNextInst())
inst->setPersistentInstructionId(PersistentInstructionId(&methodDesc, inst->getId() - irm.getMinimumInstId()));
}
}
void StackInfo::setMethodExitString(IRManager& irm)
{
ConstantAreaItem * cai=(ConstantAreaItem *)irm.getInfo("itraceMethodExitString");
if (cai!=NULL){
itraceMethodExitString=(const char*)cai->getAddress();
if (itraceMethodExitString==NULL)
itraceMethodExitString=(const char*)1;
}
}
MethodCodeSelector::MethodCodeSelector(
::Jitrino::SessionAction* _sa,
CompilationInterface& compIntfc,
MemoryManager& irMM,
MemoryManager& codeSelectorMM,
IRManager& irM)
: sa(_sa), compilationInterface(compIntfc),
irMemManager(irMM), codeSelectorMemManager(codeSelectorMM),
irManager(irM),
methodDesc(NULL),
edgeProfile(NULL)
{
ProfilingInterface* pi = irManager.getProfilingInterface();
if (pi!=NULL && pi->isProfilingEnabled(ProfileType_Edge, JITProfilingRole_GEN)) {
edgeProfile = pi->getEdgeMethodProfile(irMM, irM.getMethodDesc(), JITProfilingRole_GEN);
}
}
CfgCodeSelector::CfgCodeSelector(::Jitrino::SessionAction* sa,
CompilationInterface& compIntfc,
MethodCodeSelector& methodCodeSel,
MemoryManager& codeSelectorMM,
U_32 nNodes,
IRManager& irM
)
: numNodes(nNodes), nextNodeId(0), compilationInterface(compIntfc), methodCodeSelector(methodCodeSel),
irMemManager(irM.getMemoryManager()),
codeSelectorMemManager(codeSelectorMM), irManager(irM),
hasDispatchNodes(false), currBlock(NULL), returnOperand(0)
{
nextNodeId = 0;
nodes = new (codeSelectorMemManager) Node*[numNodes];
U_32 i;
for (i = 0; i < numNodes; i++)
nodes[i] = NULL;
InstCodeSelector::onCFGInit(irManager);
flags.useInternalHelpersForInteger2FloatConv = sa->getBoolArg("useInternalHelpersForInteger2FloatConv", false);
flags.slowLdString = sa->getBoolArg("SlowLdString", false);
}
static void doUnroll(MemoryManager& mm, IRManager& irm, const LoopUnrollInfo* info, const UnrollFlags& flags) {
//unroll algorithm does the following
//before:
// loopOrig {
// bodyA
// check(idxOpnd,limitOpnd)
// bodyB
// }
//after:
// unrolledIncOpnd = unrollCount * idx->increment
// unrolledLimitOpnd = limitOpnd-unrolledIncOpnd;
// bodyA
// loopUnrolled {
// check(idxOpnd,unrolledLimitOpnd)
// bodyB
// bodyA
// bodyB
// ...
// bodyA
// }
// loopEpilogue {
// check(idxOpnd,limitOpnd)
// bodyB
// bodyA
// }
//
//where:
// bodyA - all nodes of the same loop accessible from checkNode via incoming edges
// bodyB - all nodes except bodyA and checkNode
ControlFlowGraph& cfg = irm.getFlowGraph();
LoopTree* lt = cfg.getLoopTree();
InstFactory& instFactory = irm.getInstFactory();
OpndManager& opndManager = irm.getOpndManager();
Type* opType = info->getLimitOpnd()->getType();
// printf("UNROLL\n");
//STEP 0: cache all data needed
assert(info->unrollCount >= 1);
Node* origHeader = info->header;
assert(origHeader->getInDegree() == 2); //loop is normalized
OptPass::computeLoops(irm);//recompute loop info if needed
LoopNode* loopNode = lt->getLoopNode(origHeader, false);
Edge* entryEdge = origHeader->getInEdges().front();
if (lt->isBackEdge(entryEdge)) {
entryEdge = origHeader->getInEdges().back();
}
Node* origCheckNode = info->branchInst->getNode();
Edge* origLoopExitEdge = info->branchTargetIsExit ? origCheckNode->getTrueEdge() : origCheckNode->getFalseEdge();
U_32 maxNodeId = cfg.getMaxNodeId()+1; //+1 for a split check node
StlBitVector nodesInLoop(mm, maxNodeId);
{
const Nodes& loopNodes = loopNode->getNodesInLoop();
for (Nodes::const_iterator it = loopNodes.begin(), end = loopNodes.end(); it!=end; ++it) {
Node* node = *it;
nodesInLoop.setBit(node->getId());
}
}
//STEP 1: calculate bodyA nodes
BitSet aFlags(mm, maxNodeId);
calculateReachableNodesInLoop(loopNode, origHeader, origCheckNode, aFlags);
StlBitVector bodyANodes(mm, maxNodeId);
for (U_32 i=0;i<maxNodeId;i++) bodyANodes.setBit(i, aFlags.getBit(i));
//STEP 2: make checkNode a separate node, prepare loop region
bodyANodes.setBit(origCheckNode->getId(), true);
Node* checkNode = cfg.splitNodeAtInstruction(info->branchInst->prev(), true, false, instFactory.makeLabel());
nodesInLoop.setBit(checkNode->getId(), true);
Node* preCheckNode = origCheckNode;
bodyANodes.setBit(preCheckNode->getId(), true);
//STEP 3: rotate original loop
// before: {bodyA1, check , bodyB}
// after: bodyA2 {check, bodyB, bodyA1}
Edge* bodyA2ToCheckEdge = NULL;
Opnd* limitOpndInBodyA2 = NULL;
{
//WARN: info->limitOpnd and info->indexOpnd can be replaced after code duplication if promoted to vars
Opnd* limitOpndBefore = info->getLimitOpnd();
assert(preCheckNode->getOutDegree()==1 && preCheckNode->getUnconditionalEdgeTarget() == checkNode);
DefUseBuilder defUses(mm);
defUses.initialize(cfg);
OpndRenameTable opndRenameTable(mm, maxNodeId); //todo: maxNodeId is overkill estimate here
NodeRenameTable nodeRenameTable(mm, maxNodeId);
Node* bodyA2 = FlowGraph::duplicateRegion(irm, origHeader, bodyANodes, defUses, nodeRenameTable, opndRenameTable);
cfg.replaceEdgeTarget(entryEdge, bodyA2, true);
// while duplicating a region new nodes could be created and 'nodesInRegion' bitvector param is updated.
// BodyA is part of the loop -> if new nodes were created in the loop we must track them.
nodesInLoop.resize(bodyANodes.size());
for (U_32 i=0;i<bodyANodes.size();i++) nodesInLoop.setBit(i, bodyANodes.getBit(i) || nodesInLoop.getBit(i));
Node* bodyA2PreCheckNode = nodeRenameTable.getMapping(preCheckNode);
assert(bodyA2PreCheckNode->getOutDegree()==1 && bodyA2PreCheckNode->getUnconditionalEdgeTarget() == checkNode);
bodyA2ToCheckEdge = bodyA2PreCheckNode->getUnconditionalEdge();
limitOpndInBodyA2 = limitOpndBefore;
if (nodeRenameTable.getMapping(limitOpndBefore->getInst()->getNode())!=NULL) {
limitOpndInBodyA2 = opndRenameTable.getMapping(limitOpndBefore);
}
assert(limitOpndInBodyA2!=NULL);
}
//STEP 4: prepare epilogue loop: {check, bodyB, bodyA}
Node* epilogueLoopHead = NULL;
{
DefUseBuilder defUses(mm);
defUses.initialize(cfg);
OpndRenameTable opndRenameTable(mm, maxNodeId); //todo: maxNodeId is overkill estimate here
NodeRenameTable nodeRenameTable(mm, maxNodeId);
epilogueLoopHead = FlowGraph::duplicateRegion(irm, checkNode, nodesInLoop, defUses, nodeRenameTable, opndRenameTable);
cfg.replaceEdgeTarget(origLoopExitEdge, epilogueLoopHead, true);
}
//STEP 5: prepare unrolledLimitOpnd and replace it in original loop's check
{
Node* unrolledPreheader = cfg.spliceBlockOnEdge(bodyA2ToCheckEdge, instFactory.makeLabel());
Opnd* unrolledIncOpnd = opndManager.createSsaTmpOpnd(opType);
unrolledPreheader->appendInst(instFactory.makeLdConst(unrolledIncOpnd, info->increment * info->unrollCount));
Opnd* unrolledLimitOpnd = opndManager.createSsaTmpOpnd(opType);
Modifier mod = Modifier(SignedOp)|Modifier(Strict_No)|Modifier(Overflow_None)|Modifier(Exception_Never);
unrolledPreheader->appendInst(instFactory.makeSub(mod, unrolledLimitOpnd, limitOpndInBodyA2, unrolledIncOpnd));
info->branchInst->setSrc(info->branchLimitOpndPos, unrolledLimitOpnd);
}
DefUseBuilder defUses(mm);
defUses.initialize(cfg);
//STEP 6: unroll original loop and remove all checks in duplicated bodies
{
Edge* backedge = preCheckNode->getUnconditionalEdge();
for (int i=1;i<info->unrollCount;i++) {
OpndRenameTable opndRenameTable(mm, maxNodeId);
NodeRenameTable nodeRenameTable(mm, maxNodeId);
Node* unrolledRegionHeader = FlowGraph::duplicateRegion(irm, checkNode, nodesInLoop, defUses, nodeRenameTable, opndRenameTable);
cfg.replaceEdgeTarget(backedge, unrolledRegionHeader, true);
Node* newTail = nodeRenameTable.getMapping(preCheckNode);
assert(newTail->getOutDegree()==1 );
backedge = newTail->getUnconditionalEdge();
cfg.replaceEdgeTarget(backedge, checkNode, true);
//remove check from duplicated code
Node* duplicateCheckNode = nodeRenameTable.getMapping(checkNode);
assert(duplicateCheckNode->getOutDegree()==2);
Edge* exitEdge = info->branchTargetIsExit ? duplicateCheckNode->getTrueEdge() : duplicateCheckNode->getFalseEdge();
duplicateCheckNode->getLastInst()->unlink();
cfg.removeEdge(exitEdge);
}
}
//STEP 7: make old loop colder
if (cfg.hasEdgeProfile()) {
Edge* epilogueExit = info->branchTargetIsExit ? epilogueLoopHead->getTrueEdge() : epilogueLoopHead->getFalseEdge();
epilogueExit->setEdgeProb(epilogueExit->getEdgeProb() * 5);
}
}
void LoopUnrollPass::_run(IRManager& irm) {
const UnrollFlags& flags = ((LoopUnrollAction*)getAction())->getFlags();
OptPass::computeDominatorsAndLoops(irm);
ControlFlowGraph& cfg = irm.getFlowGraph();
LoopTree* lt = cfg.getLoopTree();
if (!lt->hasLoops()) {
return;
}
MemoryManager mm("loopUnrollMM");
UnrollInfos loopsToUnroll(mm);
findLoopsToUnroll(mm, irm, loopsToUnroll, flags);
if (loopsToUnroll.empty()) {
if (Log::isEnabled()) Log::out() << "No candidates found to unroll"<<std::endl;
return;
}
if (Log::isEnabled()) {
Log::out()<<"Loops to unroll before filtering:"<<std::endl;
for (UnrollInfos::const_iterator it = loopsToUnroll.begin(), end = loopsToUnroll.end();it!=end; ++it) {
const LoopUnrollInfo* info = *it;
info->print(Log::out()); Log::out()<<std::endl;
}
}
bool hasProfile = cfg.hasEdgeProfile();
//filter out that can't be unrolled, calculate BodyA and BodyB
BitSet bodyANodes(mm, cfg.getMaxNodeId()), bodyBNodes(mm, cfg.getMaxNodeId());
for (UnrollInfos::iterator it = loopsToUnroll.begin(), end = loopsToUnroll.end();it!=end; ++it) {
LoopUnrollInfo* info = *it;
if (info == NULL) {
continue;
}
if (!info->doUnroll) {
*it=NULL;
continue;
}
Node* header=info->header;
LoopNode* loopHeader = lt->getLoopNode(header, false);
assert(loopHeader->getHeader() == header);
Node* checkNode = info->branchInst->getNode();
bodyANodes.clear();
bodyBNodes.clear();
calculateReachableNodesInLoop(loopHeader, loopHeader->getHeader(), checkNode, bodyANodes);
calculateReachableNodesInLoop(loopHeader, checkNode, NULL, bodyBNodes);
bodyANodes.intersectWith(bodyBNodes);
bool checkNodeIsJunctionPoint = bodyANodes.isEmpty();
if (!checkNodeIsJunctionPoint) {
if (Log::isEnabled()) {
Log::out()<<"Check node is not a junction point -> removing from the list: branch inst id=I"<<info->branchInst->getId()<<std::endl;
}
*it=NULL;
continue;
}
//check if branch semantic is OK
ComparisonModifier cmpMod = info->branchInst->getModifier().getComparisonModifier();
if (cmpMod!=Cmp_GT && cmpMod!=Cmp_GTE && cmpMod!=Cmp_GT_Un && cmpMod!=Cmp_GTE_Un) {
if (Log::isEnabled()) {
Log::out()<<"Branch is not a range comparison -> removing from the list: branch inst id=I"<<info->branchInst->getId()<<std::endl;
}
*it=NULL;
continue;
}
//check config settings
bool failed = false;
int nodesInLoop = (int)loopHeader->getNodesInLoop().size();
const char* reason = "unknown";
if (nodesInLoop > flags.largeLoopSize) {
reason = "loop is too large";
failed = true;
} else if (hasProfile) {
int headHotness = (int)(header->getExecCount()*100.0 / cfg.getEntryNode()->getExecCount());
int minHeaderHotness= nodesInLoop <= flags.smallLoopSize ? flags.smallLoopHotness :
nodesInLoop <= flags.mediumLoopSize ? flags.mediumLoopHotness : flags.largeLoopHotness;
info->unrollCount = nodesInLoop <= flags.smallLoopSize ? flags.smallLoopUnrollCount :
nodesInLoop <= flags.mediumLoopSize? flags.mediumLoopUnrollCount: flags.largeLoopUnrollCount;
failed = headHotness < minHeaderHotness || info->unrollCount < 1;
if (failed) {
reason = "loop is too cold";
}
}
if (failed) {
if (Log::isEnabled()) {
Log::out()<<"Loop does not match unroll configuration ("<<reason<<") -> removing from the list: branch inst id=I"<<info->branchInst->getId()<<std::endl;
}
*it=NULL;
}
}
//filter out loops with multiple exits
for (UnrollInfos::iterator it1 = loopsToUnroll.begin(), end = loopsToUnroll.end();it1!=end; ++it1) {
const LoopUnrollInfo* info1 = *it1;
if (info1== NULL) {
continue;
}
Node* header=info1->header;
for (UnrollInfos::iterator it2 = it1+1; it2!=end; ++it2) {
const LoopUnrollInfo* info2 = *it2;
if (info2!=NULL && header==info2->header) {
if (Log::isEnabled()) {
Log::out() << "Found multiple exits:"; FlowGraph::printLabel(Log::out(), header);Log::out()<<std::endl;
}
if (hasProfile) {
Node* check1 = info1->branchInst->getNode();
Node* check2 = info2->branchInst->getNode();
if (check1->getExecCount() > check2->getExecCount()) {
*it2 = NULL;
} else {
*it1 = NULL;
}
} else { // random selection
*it2=NULL;
}
}
}
}
loopsToUnroll.erase(std::remove(loopsToUnroll.begin(), loopsToUnroll.end(), (LoopUnrollInfo*)NULL), loopsToUnroll.end());
if (loopsToUnroll.empty()) {
if (Log::isEnabled()) Log::out() << "--------No candidates to unroll left after filtering"<<std::endl;
return;
}
//dessa CFG before unrolling -> need to duplicate regions and we can do it on dessa form only today
{
SSABuilder::deconvertSSA(&cfg, irm.getOpndManager());
irm.setInSsa(false);
}
if (Log::isEnabled()) {
Log::out()<<"--------Loops to unroll after filtering : n="<<loopsToUnroll.size()<<std::endl;
for (UnrollInfos::const_iterator it = loopsToUnroll.begin(), end = loopsToUnroll.end();it!=end; ++it) {
const LoopUnrollInfo* info = *it;
info->print(Log::out()); Log::out()<<std::endl;
}
}
for (UnrollInfos::const_iterator it = loopsToUnroll.begin(), end = loopsToUnroll.end();it!=end; ++it) {
const LoopUnrollInfo* info = *it;
doUnroll(mm, irm, info, flags);
}
};
LazyExceptionOpt::LazyExceptionOpt(IRManager &ir_manager) :
irManager(ir_manager),
leMemManager("LazyExceptionOpt::doLazyExceptionOpt"),
compInterface(ir_manager.getCompilationInterface())
{
}
void
SplitSSAPass::_run(IRManager& irm) {
SSABuilder::splitSsaWebs(&irm.getFlowGraph(), irm.getOpndManager());
}
void
DeSSAPass::_run(IRManager& irm) {
SSABuilder::deconvertSSA(&irm.getFlowGraph(),irm.getOpndManager());
irm.setInSsa(false);
}
