//------------------------------split_up---------------------------------------
// Split block-local op up through the phis to empty the current block
bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
if( n->is_CFG() ) {
assert( n->in(0) != blk1, "Lousy candidate for split-if" );
return false;
}
if( get_ctrl(n) != blk1 && get_ctrl(n) != blk2 )
return false; // Not block local
if( n->is_Phi() ) return false; // Local PHIs are expected
// Recursively split-up inputs
for (uint i = 1; i < n->req(); i++) {
if( split_up( n->in(i), blk1, blk2 ) ) {
// Got split recursively and self went dead?
if (n->outcnt() == 0)
_igvn.remove_dead_node(n);
return true;
}
}
// Check for needing to clone-up a compare. Can't do that, it forces
// another (nested) split-if transform. Instead, clone it "down".
if( n->is_Cmp() ) {
assert(get_ctrl(n) == blk2 || get_ctrl(n) == blk1, "must be in block with IF");
// Check for simple Cmp/Bool/CMove which we can clone-up. Cmp/Bool/CMove
// sequence can have no other users and it must all reside in the split-if
// block. Non-simple Cmp/Bool/CMove sequences are 'cloned-down' below -
// private, per-use versions of the Cmp and Bool are made. These sink to
// the CMove block. If the CMove is in the split-if block, then in the
// next iteration this will become a simple Cmp/Bool/CMove set to clone-up.
Node *bol, *cmov;
if( !(n->outcnt() == 1 && n->unique_out()->is_Bool() &&
(bol = n->unique_out()->as_Bool()) &&
(get_ctrl(bol) == blk1 ||
get_ctrl(bol) == blk2) &&
bol->outcnt() == 1 &&
bol->unique_out()->is_CMove() &&
(cmov = bol->unique_out()->as_CMove()) &&
(get_ctrl(cmov) == blk1 ||
get_ctrl(cmov) == blk2) ) ) {
// Must clone down
#ifndef PRODUCT
if( PrintOpto && VerifyLoopOptimizations ) {
tty->print("Cloning down: ");
n->dump();
}
#endif
// Clone down any block-local BoolNode uses of this CmpNode
for (DUIterator i = n->outs(); n->has_out(i); i++) {
Node* bol = n->out(i);
assert( bol->is_Bool(), "" );
if (bol->outcnt() == 1) {
Node* use = bol->unique_out();
Node *use_c = use->is_If() ? use->in(0) : get_ctrl(use);
if (use_c == blk1 || use_c == blk2) {
continue;
}
}
if (get_ctrl(bol) == blk1 || get_ctrl(bol) == blk2) {
// Recursively sink any BoolNode
#ifndef PRODUCT
if( PrintOpto && VerifyLoopOptimizations ) {
tty->print("Cloning down: ");
bol->dump();
}
#endif
for (DUIterator_Last jmin, j = bol->last_outs(jmin); j >= jmin; --j) {
// Uses are either IfNodes or CMoves
Node* iff = bol->last_out(j);
assert( iff->in(1) == bol, "" );
// Get control block of either the CMove or the If input
Node *iff_ctrl = iff->is_If() ? iff->in(0) : get_ctrl(iff);
Node *x = bol->clone();
register_new_node(x, iff_ctrl);
_igvn.hash_delete(iff);
iff->set_req(1, x);
_igvn._worklist.push(iff);
}
_igvn.remove_dead_node( bol );
--i;
}
}
// Clone down this CmpNode
for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; --j) {
Node* bol = n->last_out(j);
assert( bol->in(1) == n, "" );
Node *x = n->clone();
register_new_node(x, get_ctrl(bol));
_igvn.hash_delete(bol);
bol->set_req(1, x);
_igvn._worklist.push(bol);
}
_igvn.remove_dead_node( n );
return true;
}
}
// See if splitting-up a Store. Any anti-dep loads must go up as
// well. An anti-dep load might be in the wrong block, because in
// this particular layout/schedule we ignored anti-deps and allow
// memory to be alive twice. This only works if we do the same
// operations on anti-dep loads as we do their killing stores.
if( n->is_Store() && n->in(MemNode::Memory)->in(0) == n->in(0) ) {
// Get store's memory slice
int alias_idx = C->get_alias_index(_igvn.type(n->in(MemNode::Address))->is_ptr());
// Get memory-phi anti-dep loads will be using
Node *memphi = n->in(MemNode::Memory);
assert( memphi->is_Phi(), "" );
// Hoist any anti-dep load to the splitting block;
// it will then "split-up".
for (DUIterator_Fast imax,i = memphi->fast_outs(imax); i < imax; i++) {
Node *load = memphi->fast_out(i);
if( load->is_Load() && alias_idx == C->get_alias_index(_igvn.type(load->in(MemNode::Address))->is_ptr()) )
set_ctrl(load,blk1);
}
}
// Found some other Node; must clone it up
#ifndef PRODUCT
if( PrintOpto && VerifyLoopOptimizations ) {
tty->print("Cloning up: ");
n->dump();
}
#endif
// Now actually split-up this guy. One copy per control path merging.
Node *phi = PhiNode::make_blank(blk1, n);
for( uint j = 1; j < blk1->req(); j++ ) {
Node *x = n->clone();
if( n->in(0) && n->in(0) == blk1 )
x->set_req( 0, blk1->in(j) );
for( uint i = 1; i < n->req(); i++ ) {
Node *m = n->in(i);
if( get_ctrl(m) == blk1 ) {
assert( m->in(0) == blk1, "" );
x->set_req( i, m->in(j) );
}
}
register_new_node( x, blk1->in(j) );
phi->init_req( j, x );
}
// Announce phi to optimizer
register_new_node(phi, blk1);
// Remove cloned-up value from optimizer; use phi instead
_igvn.hash_delete(n);
_igvn.subsume_node( n, phi );
// (There used to be a self-recursive call to split_up() here,
// but it is not needed. All necessary forward walking is done
// by do_split_if() below.)
return true;
}
//------------------------------do_unswitching-----------------------------
// Clone loop with an invariant test (that does not exit) and
// insert a clone of the test that selects which version to
// execute.
void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) {
// Find first invariant test that doesn't exit the loop
LoopNode *head = loop->_head->as_Loop();
IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
assert(unswitch_iff != NULL, "should be at least one");
// Need to revert back to normal loop
if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) {
head->as_CountedLoop()->set_normal_loop();
}
ProjNode* proj_true = create_slow_version_of_loop(loop, old_new);
assert(proj_true->is_IfTrue() && proj_true->unique_ctrl_out() == head, "by construction");
// Increment unswitch count
LoopNode* head_clone = old_new[head->_idx]->as_Loop();
int nct = head->unswitch_count() + 1;
head->set_unswitch_count(nct);
head_clone->set_unswitch_count(nct);
// Add test to new "if" outside of loop
IfNode* invar_iff = proj_true->in(0)->as_If();
Node* invar_iff_c = invar_iff->in(0);
BoolNode* bol = unswitch_iff->in(1)->as_Bool();
invar_iff->set_req(1, bol);
invar_iff->_prob = unswitch_iff->_prob;
ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
// Hoist invariant casts out of each loop to the appropriate
// control projection.
Node_List worklist;
for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
// Copy to a worklist for easier manipulation
for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
Node* use = proj->fast_out(j);
if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
worklist.push(use);
}
}
ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
while (worklist.size() > 0) {
Node* use = worklist.pop();
Node* nuse = use->clone();
nuse->set_req(0, invar_proj);
_igvn.hash_delete(use);
use->set_req(1, nuse);
_igvn._worklist.push(use);
register_new_node(nuse, invar_proj);
// Same for the clone
Node* use_clone = old_new[use->_idx];
_igvn.hash_delete(use_clone);
use_clone->set_req(1, nuse);
_igvn._worklist.push(use_clone);
}
}
// Hardwire the control paths in the loops into if(true) and if(false)
_igvn.hash_delete(unswitch_iff);
short_circuit_if(unswitch_iff, proj_true);
_igvn._worklist.push(unswitch_iff);
IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
_igvn.hash_delete(unswitch_iff_clone);
short_circuit_if(unswitch_iff_clone, proj_false);
_igvn._worklist.push(unswitch_iff_clone);
// Reoptimize loops
loop->record_for_igvn();
for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
Node *n = loop->_body[i];
Node *n_clone = old_new[n->_idx];
_igvn._worklist.push(n_clone);
}
#ifndef PRODUCT
if (TraceLoopUnswitching) {
tty->print_cr("Loop unswitching orig: %d @ %d new: %d @ %d",
head->_idx, unswitch_iff->_idx,
old_new[head->_idx]->_idx, unswitch_iff_clone->_idx);
}
#endif
C->set_major_progress();
}
// We must be at the merge point which post-dominates 'new_false' and
// 'new_true'. Figure out which edges into the RegionNode eventually lead up
// to false and which to true. Put in a PhiNode to merge values; plug in
// the appropriate false-arm or true-arm values. If some path leads to the
// original IF, then insert a Phi recursively.
Node *PhaseIdealLoop::spinup( Node *iff_dom, Node *new_false, Node *new_true, Node *use_blk, Node *def, small_cache *cache ) {
if (use_blk->is_top()) // Handle dead uses
return use_blk;
Node *prior_n = (Node*)0xdeadbeef;
Node *n = use_blk; // Get path input
assert( use_blk != iff_dom, "" );
// Here's the "spinup" the dominator tree loop. Do a cache-check
// along the way, in case we've come this way before.
while( n != iff_dom ) { // Found post-dominating point?
prior_n = n;
n = idom(n); // Search higher
Node *s = cache->probe( prior_n ); // Check cache
if( s ) return s; // Cache hit!
}
Node *phi_post;
if( prior_n == new_false || prior_n == new_true ) {
phi_post = def->clone();
phi_post->set_req(0, prior_n );
register_new_node(phi_post, prior_n);
} else {
// This method handles both control uses (looking for Regions) or data
// uses (looking for Phis). If looking for a control use, then we need
// to insert a Region instead of a Phi; however Regions always exist
// previously (the hash_find_insert below would always hit) so we can
// return the existing Region.
if( def->is_CFG() ) {
phi_post = prior_n; // If looking for CFG, return prior
} else {
assert( def->is_Phi(), "" );
assert( prior_n->is_Region(), "must be a post-dominating merge point" );
// Need a Phi here
phi_post = PhiNode::make_blank(prior_n, def);
// Search for both true and false on all paths till find one.
for( uint i = 1; i < phi_post->req(); i++ ) // For all paths
phi_post->init_req( i, spinup( iff_dom, new_false, new_true, prior_n->in(i), def, cache ) );
Node *t = _igvn.hash_find_insert(phi_post);
if( t ) { // See if we already have this one
// phi_post will not be used, so kill it
_igvn.remove_dead_node(phi_post);
phi_post->destruct();
phi_post = t;
} else {
register_new_node( phi_post, prior_n );
}
}
}
// Update cache everywhere
prior_n = (Node*)0xdeadbeef; // Reset IDOM walk
n = use_blk; // Get path input
// Spin-up the idom tree again, basically doing path-compression.
// Insert cache entries along the way, so that if we ever hit this
// point in the IDOM tree again we'll stop immediately on a cache hit.
while( n != iff_dom ) { // Found post-dominating point?
prior_n = n;
n = idom(n); // Search higher
cache->lru_insert( prior_n, phi_post ); // Fill cache
} // End of while not gone high enough
return phi_post;
}
