LoopNode* PhaseIdealLoop::create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk) {
Node_List old_new;
LoopNode* head = loop->_head->as_Loop();
bool counted_loop = head->is_CountedLoop();
Node* entry = head->in(LoopNode::EntryControl);
_igvn.rehash_node_delayed(entry);
IdealLoopTree* outer_loop = loop->_parent;
ConINode* const_1 = _igvn.intcon(1);
set_ctrl(const_1, C->root());
IfNode* iff = new IfNode(entry, const_1, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
// Clone the loop body. The clone becomes the fast loop. The
// original pre-header will (illegally) have 3 control users
// (old & new loops & new if).
clone_loop(loop, old_new, dom_depth(head), iff);
assert(old_new[head->_idx]->is_Loop(), "" );
LoopNode* slow_head = old_new[head->_idx]->as_Loop();
#ifndef PRODUCT
if (TraceLoopOpts) {
tty->print_cr("PhaseIdealLoop::create_reserve_version_of_loop:");
tty->print("\t iff = %d, ", iff->_idx); iff->dump();
tty->print("\t iffast = %d, ", iffast->_idx); iffast->dump();
tty->print("\t ifslow = %d, ", ifslow->_idx); ifslow->dump();
tty->print("\t before replace_input_of: head = %d, ", head->_idx); head->dump();
tty->print("\t before replace_input_of: slow_head = %d, ", slow_head->_idx); slow_head->dump();
}
#endif
// Fast (true) control
_igvn.replace_input_of(head, LoopNode::EntryControl, iffast);
// Slow (false) control
_igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow);
recompute_dom_depth();
lk->set_iff(iff);
#ifndef PRODUCT
if (TraceLoopOpts ) {
tty->print("\t after replace_input_of: head = %d, ", head->_idx); head->dump();
tty->print("\t after replace_input_of: slow_head = %d, ", slow_head->_idx); slow_head->dump();
}
#endif
return slow_head->as_Loop();
}
//------------------------------split_thru_region------------------------------
// Split Node 'n' through merge point.
Node *PhaseIdealLoop::split_thru_region( Node *n, Node *region ) {
uint wins = 0;
assert( n->is_CFG(), "" );
assert( region->is_Region(), "" );
Node *r = new (C, region->req()) RegionNode( region->req() );
IdealLoopTree *loop = get_loop( n );
for( uint i = 1; i < region->req(); i++ ) {
Node *x = n->clone();
Node *in0 = n->in(0);
if( in0->in(0) == region ) x->set_req( 0, in0->in(i) );
for( uint j = 1; j < n->req(); j++ ) {
Node *in = n->in(j);
if( get_ctrl(in) == region )
x->set_req( j, in->in(i) );
}
_igvn.register_new_node_with_optimizer(x);
set_loop(x, loop);
set_idom(x, x->in(0), dom_depth(x->in(0))+1);
r->init_req(i, x);
}
// Record region
r->set_req(0,region); // Not a TRUE RegionNode
_igvn.register_new_node_with_optimizer(r);
set_loop(r, loop);
if( !loop->_child )
loop->_body.push(r);
return r;
}
//-------------------------------register_control-------------------------
void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred) {
assert(n->is_CFG(), "must be control node");
_igvn.register_new_node_with_optimizer(n);
loop->_body.push(n);
set_loop(n, loop);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
set_idom(n, pred, dom_depth(pred));
}
}
//-------------------------create_slow_version_of_loop------------------------
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
// Return control projection of the entry to the fast version.
ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new,
int opcode) {
LoopNode* head = loop->_head->as_Loop();
bool counted_loop = head->is_CountedLoop();
Node* entry = head->in(LoopNode::EntryControl);
_igvn.rehash_node_delayed(entry);
IdealLoopTree* outer_loop = loop->_parent;
Node *cont = _igvn.intcon(1);
set_ctrl(cont, C->root());
Node* opq = new Opaque1Node(C, cont);
register_node(opq, outer_loop, entry, dom_depth(entry));
Node *bol = new Conv2BNode(opq);
register_node(bol, outer_loop, entry, dom_depth(entry));
IfNode* iff = (opcode == Op_RangeCheck) ? new RangeCheckNode(entry, bol, PROB_MAX, COUNT_UNKNOWN) :
new IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
// Clone the loop body. The clone becomes the fast loop. The
// original pre-header will (illegally) have 3 control users
// (old & new loops & new if).
clone_loop(loop, old_new, dom_depth(head), iff);
assert(old_new[head->_idx]->is_Loop(), "" );
// Fast (true) control
Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop);
_igvn.replace_input_of(head, LoopNode::EntryControl, iffast_pred);
set_idom(head, iffast_pred, dom_depth(head));
// Slow (false) control
Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop);
LoopNode* slow_head = old_new[head->_idx]->as_Loop();
_igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow_pred);
set_idom(slow_head, ifslow_pred, dom_depth(slow_head));
recompute_dom_depth();
return iffast;
}
//-------------------------create_slow_version_of_loop------------------------
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
// Return control projection of the entry to the fast version.
ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new) {
LoopNode* head = loop->_head->as_Loop();
Node* entry = head->in(LoopNode::EntryControl);
_igvn.hash_delete(entry);
_igvn._worklist.push(entry);
IdealLoopTree* outer_loop = loop->_parent;
Node *cont = _igvn.intcon(1);
set_ctrl(cont, C->root());
Node* opq = new (C, 2) Opaque1Node(C, cont);
register_node(opq, outer_loop, entry, dom_depth(entry));
Node *bol = new (C, 2) Conv2BNode(opq);
register_node(bol, outer_loop, entry, dom_depth(entry));
IfNode* iff = new (C, 2) IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new (C, 1) IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new (C, 1) IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
// Clone the loop body. The clone becomes the fast loop. The
// original pre-header will (illegally) have 2 control users (old & new loops).
clone_loop(loop, old_new, dom_depth(head), iff);
assert(old_new[head->_idx]->is_Loop(), "" );
// Fast (true) control
_igvn.hash_delete(head);
head->set_req(LoopNode::EntryControl, iffast);
set_idom(head, iffast, dom_depth(head));
_igvn._worklist.push(head);
// Slow (false) control
LoopNode* slow_head = old_new[head->_idx]->as_Loop();
_igvn.hash_delete(slow_head);
slow_head->set_req(LoopNode::EntryControl, ifslow);
set_idom(slow_head, ifslow, dom_depth(slow_head));
_igvn._worklist.push(slow_head);
recompute_dom_depth();
return iffast;
}
//------------------------------create_new_if_for_predicate------------------------
// create a new if above the uct_if_pattern for the predicate to be promoted.
//
// before after
// ---------- ----------
// ctrl ctrl
// | |
// | |
// v v
// iff new_iff
// / \ / \
// / \ / \
// v v v v
// uncommon_proj cont_proj if_uct if_cont
// \ | | | |
// \ | | | |
// v v v | v
// rgn loop | iff
// | | / \
// | | / \
// v | v v
// uncommon_trap | uncommon_proj cont_proj
// \ \ | |
// \ \ | |
// v v v v
// rgn loop
// |
// |
// v
// uncommon_trap
//
//
// We will create a region to guard the uct call if there is no one there.
// The true projecttion (if_cont) of the new_iff is returned.
// This code is also used to clone predicates to clonned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason) {
assert(is_uncommon_trap_if_pattern(cont_proj, reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
IdealLoopTree* loop = get_loop(call);
rgn = new (C) RegionNode(1);
rgn->add_req(uncommon_proj);
register_control(rgn, loop, uncommon_proj);
_igvn.hash_delete(call);
call->set_req(0, rgn);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
set_idom(call, rgn, dom_depth(rgn));
}
} else {
// Find region's edge corresponding to uncommon_proj
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
}
Node* entry = iff->in(0);
if (new_entry != NULL) {
// Clonning the predicate to new location.
entry = new_entry;
}
// Create new_iff
IdealLoopTree* lp = get_loop(entry);
IfNode *new_iff = iff->clone()->as_If();
new_iff->set_req(0, entry);
register_control(new_iff, lp, entry);
Node *if_cont = new (C) IfTrueNode(new_iff);
Node *if_uct = new (C) IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
// Swap
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_control(if_cont, lp, new_iff);
register_control(if_uct, get_loop(rgn), new_iff);
// if_uct to rgn
_igvn.hash_delete(rgn);
rgn->add_req(if_uct);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != NULL) {
Node* ridom = idom(rgn);
Node* nrdom = dom_lca(ridom, new_iff);
set_idom(rgn, nrdom, dom_depth(rgn));
}
// If rgn has phis add new edges which has the same
// value as on original uncommon_proj pass.
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
_igvn.rehash_node_delayed(use);
use->add_req(use->in(proj_index));
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
if (new_entry == NULL) {
// Attach if_cont to iff
_igvn.hash_delete(iff);
iff->set_req(0, if_cont);
if (_idom != NULL) {
set_idom(iff, if_cont, dom_depth(iff));
}
}
return if_cont->as_Proj();
}
//------------------------------do_split_if------------------------------------
// Found an If getting its condition-code input from a Phi in the same block.
// Split thru the Region.
void PhaseIdealLoop::do_split_if( Node *iff ) {
#ifndef PRODUCT
if( PrintOpto && VerifyLoopOptimizations )
tty->print_cr("Split-if");
#endif
C->set_major_progress();
Node *region = iff->in(0);
Node *region_dom = idom(region);
// We are going to clone this test (and the control flow with it) up through
// the incoming merge point. We need to empty the current basic block.
// Clone any instructions which must be in this block up through the merge
// point.
DUIterator i, j;
bool progress = true;
while (progress) {
progress = false;
for (i = region->outs(); region->has_out(i); i++) {
Node* n = region->out(i);
if( n == region ) continue;
// The IF to be split is OK.
if( n == iff ) continue;
if( !n->is_Phi() ) { // Found pinned memory op or such
if (split_up(n, region, iff)) {
i = region->refresh_out_pos(i);
progress = true;
}
continue;
}
assert( n->in(0) == region, "" );
// Recursively split up all users of a Phi
for (j = n->outs(); n->has_out(j); j++) {
Node* m = n->out(j);
// If m is dead, throw it away, and declare progress
if (_nodes[m->_idx] == NULL) {
_igvn.remove_dead_node(m);
// fall through
}
else if (m != iff && split_up(m, region, iff)) {
// fall through
} else {
continue;
}
// Something unpredictable changed.
// Tell the iterators to refresh themselves, and rerun the loop.
i = region->refresh_out_pos(i);
j = region->refresh_out_pos(j);
progress = true;
}
}
}
// Now we have no instructions in the block containing the IF.
// Split the IF.
Node *new_iff = split_thru_region( iff, region );
// Replace both uses of 'new_iff' with Regions merging True/False
// paths. This makes 'new_iff' go dead.
Node *old_false, *old_true;
Node *new_false, *new_true;
for (DUIterator_Last j2min, j2 = iff->last_outs(j2min); j2 >= j2min; --j2) {
Node *ifp = iff->last_out(j2);
assert( ifp->Opcode() == Op_IfFalse || ifp->Opcode() == Op_IfTrue, "" );
ifp->set_req(0, new_iff);
Node *ifpx = split_thru_region( ifp, region );
// Replace 'If' projection of a Region with a Region of
// 'If' projections.
ifpx->set_req(0, ifpx); // A TRUE RegionNode
// Setup dominator info
set_idom(ifpx, region_dom, dom_depth(region_dom) + 1);
// Check for splitting loop tails
if( get_loop(iff)->tail() == ifp )
get_loop(iff)->_tail = ifpx;
// Replace in the graph with lazy-update mechanism
new_iff->set_req(0, new_iff); // hook self so it does not go dead
lazy_replace_proj( ifp, ifpx );
new_iff->set_req(0, region);
// Record bits for later xforms
if( ifp->Opcode() == Op_IfFalse ) {
old_false = ifp;
new_false = ifpx;
} else {
old_true = ifp;
new_true = ifpx;
}
}
_igvn.remove_dead_node(new_iff);
// Lazy replace IDOM info with the region's dominator
lazy_replace( iff, region_dom );
// Now make the original merge point go dead, by handling all its uses.
small_cache region_cache;
// Preload some control flow in region-cache
region_cache.lru_insert( new_false, new_false );
region_cache.lru_insert( new_true , new_true );
// Now handle all uses of the splitting block
for (DUIterator_Last kmin, k = region->last_outs(kmin); k >= kmin; --k) {
Node* phi = region->last_out(k);
if( !phi->in(0) ) { // Dead phi? Remove it
_igvn.remove_dead_node(phi);
continue;
}
assert( phi->in(0) == region, "" );
if( phi == region ) { // Found the self-reference
phi->set_req(0, NULL);
continue; // Break the self-cycle
}
// Expected common case: Phi hanging off of Region
if( phi->is_Phi() ) {
// Need a per-def cache. Phi represents a def, so make a cache
small_cache phi_cache;
// Inspect all Phi uses to make the Phi go dead
for (DUIterator_Last lmin, l = phi->last_outs(lmin); l >= lmin; --l) {
Node* use = phi->last_out(l);
// Compute the new DEF for this USE. New DEF depends on the path
// taken from the original DEF to the USE. The new DEF may be some
// collection of PHI's merging values from different paths. The Phis
// inserted depend only on the location of the USE. We use a
// 2-element cache to handle multiple uses from the same block.
handle_use( use, phi, &phi_cache, region_dom, new_false, new_true, old_false, old_true );
} // End of while phi has uses
// Because handle_use might relocate region->_out,
// we must refresh the iterator.
k = region->last_outs(kmin);
// Remove the dead Phi
_igvn.remove_dead_node( phi );
} else {
// Random memory op guarded by Region. Compute new DEF for USE.
handle_use( phi, region, ®ion_cache, region_dom, new_false, new_true, old_false, old_true );
}
} // End of while merge point has phis
// Any leftover bits in the splitting block must not have depended on local
// Phi inputs (these have already been split-up). Hence it's safe to hoist
// these guys to the dominating point.
lazy_replace( region, region_dom );
#ifndef PRODUCT
if( VerifyLoopOptimizations ) verify();
#endif
}