//------------------------------add--------------------------------------------
// Add edge between vertices a & b. These are sorted (triangular matrix),
// then the smaller number is inserted in the larger numbered array.
int PhaseIFG::add_edge( uint a, uint b ) {
lrgs(a).invalid_degree();
lrgs(b).invalid_degree();
// Sort a and b, so that a is bigger
assert( !_is_square, "only on triangular" );
if( a < b ) { uint tmp = a; a = b; b = tmp; }
return _adjs[a].insert( b );
}
//------------------------------interfere_with_live----------------------------
// Interfere this register with everything currently live. Use the RegMasks
// to trim the set of possible interferences. Return a count of register-only
// interferences as an estimate of register pressure.
void PhaseChaitin::interfere_with_live( uint r, IndexSet *liveout ) {
uint retval = 0;
// Interfere with everything live.
const RegMask &rm = lrgs(r).mask();
// Check for interference by checking overlap of regmasks.
// Only interfere if acceptable register masks overlap.
IndexSetIterator elements(liveout);
uint l;
while( (l = elements.next()) != 0 )
if( rm.overlap( lrgs(l).mask() ) )
_ifg->add_edge( r, l );
}
//------------------------------count_float_pressure---------------------------
uint PhaseChaitin::count_float_pressure( IndexSet *liveout ) {
IndexSetIterator elements(liveout);
uint lidx;
uint cnt = 0;
while ((lidx = elements.next()) != 0) {
if( lrgs(lidx).mask().is_UP() &&
lrgs(lidx).mask_size() &&
lrgs(lidx)._is_float )
cnt += lrgs(lidx).reg_pressure();
}
return cnt;
}
//------------------------------count_int_pressure-----------------------------
uint PhaseChaitin::count_int_pressure( IndexSet *liveout ) {
IndexSetIterator elements(liveout);
uint lidx;
uint cnt = 0;
while ((lidx = elements.next()) != 0) {
if( lrgs(lidx).mask().is_UP() &&
lrgs(lidx).mask_size() &&
!lrgs(lidx)._is_float &&
lrgs(lidx).mask().overlap(*Matcher::idealreg2regmask[Op_RegI]) )
cnt += lrgs(lidx).reg_pressure();
}
return cnt;
}
//------------------------------Union------------------------------------------
// Union edges of B into A
void PhaseIFG::Union( uint a, uint b ) {
assert( _is_square, "only on square" );
IndexSet *A = &_adjs[a];
IndexSetIterator b_elements(&_adjs[b]);
uint datum;
while ((datum = b_elements.next()) != 0) {
if(A->insert(datum)) {
_adjs[datum].insert(a);
lrgs(a).invalid_degree();
lrgs(datum).invalid_degree();
}
}
}
//------------------------------update_ifg-------------------------------------
void PhaseConservativeCoalesce::update_ifg(uint lr1, uint lr2, IndexSet *n_lr1, IndexSet *n_lr2) {
// Some original neighbors of lr1 might have gone away
// because the constrained register mask prevented them.
// Remove lr1 from such neighbors.
IndexSetIterator one(n_lr1);
uint neighbor;
LRG &lrg1 = lrgs(lr1);
while ((neighbor = one.next()) != 0)
if( !_ulr.member(neighbor) )
if( _phc._ifg->neighbors(neighbor)->remove(lr1) )
lrgs(neighbor).inc_degree( -lrg1.compute_degree(lrgs(neighbor)) );
// lr2 is now called (coalesced into) lr1.
// Remove lr2 from the IFG.
IndexSetIterator two(n_lr2);
LRG &lrg2 = lrgs(lr2);
while ((neighbor = two.next()) != 0)
if( _phc._ifg->neighbors(neighbor)->remove(lr2) )
lrgs(neighbor).inc_degree( -lrg2.compute_degree(lrgs(neighbor)) );
// Some neighbors of intermediate copies now interfere with the
// combined live range.
IndexSetIterator three(&_ulr);
while ((neighbor = three.next()) != 0)
if( _phc._ifg->neighbors(neighbor)->insert(lr1) )
lrgs(neighbor).inc_degree( lrg1.compute_degree(lrgs(neighbor)) );
}
//------------------------------remove_node------------------------------------
// Yank a Node and all connected edges from the IFG. Return a
// list of neighbors (edges) yanked.
IndexSet *PhaseIFG::remove_node( uint a ) {
assert( _is_square, "only on square" );
assert( !_yanked->test(a), "" );
_yanked->set(a);
// I remove the LRG from all neighbors.
IndexSetIterator elements(&_adjs[a]);
LRG &lrg_a = lrgs(a);
uint datum;
while ((datum = elements.next()) != 0) {
_adjs[datum].remove(a);
lrgs(datum).inc_degree( -lrg_a.compute_degree(lrgs(datum)) );
}
return neighbors(a);
}
//------------------------------effective_degree-------------------------------
// Compute effective degree for this live range. If both live ranges are
// aligned-adjacent powers-of-2 then we use the MAX size. If either is
// mis-aligned (or for Fat-Projections, not-adjacent) then we have to
// MULTIPLY the sizes. Inspect Brigg's thesis on register pairs to see why
// this is so.
int PhaseIFG::effective_degree( uint lidx ) const {
int eff = 0;
int num_regs = lrgs(lidx).num_regs();
int fat_proj = lrgs(lidx)._fat_proj;
IndexSet *s = neighbors(lidx);
IndexSetIterator elements(s);
uint nidx;
while((nidx = elements.next()) != 0) {
LRG &lrgn = lrgs(nidx);
int nregs = lrgn.num_regs();
eff += (fat_proj || lrgn._fat_proj) // either is a fat-proj?
? (num_regs * nregs) // then use product
: MAX2(num_regs,nregs); // else use max
}
return eff;
}
//------------------------------compute_separating_interferences---------------
// Factored code from copy_copy that computes extra interferences from
// lengthening a live range by double-coalescing.
uint PhaseConservativeCoalesce::compute_separating_interferences(Node *dst_copy, Node *src_copy, Block *b, uint bindex, RegMask &rm, uint reg_degree, uint rm_size, uint lr1, uint lr2 ) {
assert(!lrgs(lr1)._fat_proj, "cannot coalesce fat_proj");
assert(!lrgs(lr2)._fat_proj, "cannot coalesce fat_proj");
Node *prev_copy = dst_copy->in(dst_copy->is_Copy());
Block *b2 = b;
uint bindex2 = bindex;
while( 1 ) {
// Find previous instruction
bindex2--; // Chain backwards 1 instruction
while( bindex2 == 0 ) { // At block start, find prior block
assert( b2->num_preds() == 2, "cannot double coalesce across c-flow" );
b2 = _phc._cfg._bbs[b2->pred(1)->_idx];
bindex2 = b2->end_idx()-1;
}
// Get prior instruction
assert(bindex2 < b2->_nodes.size(), "index out of bounds");
Node *x = b2->_nodes[bindex2];
if( x == prev_copy ) { // Previous copy in copy chain?
if( prev_copy == src_copy)// Found end of chain and all interferences
break; // So break out of loop
// Else work back one in copy chain
prev_copy = prev_copy->in(prev_copy->is_Copy());
} else { // Else collect interferences
uint lidx = _phc.Find(x);
// Found another def of live-range being stretched?
if( lidx == lr1 ) return max_juint;
if( lidx == lr2 ) return max_juint;
// If we attempt to coalesce across a bound def
if( lrgs(lidx).is_bound() ) {
// Do not let the coalesced LRG expect to get the bound color
rm.SUBTRACT( lrgs(lidx).mask() );
// Recompute rm_size
rm_size = rm.Size();
//if( rm._flags ) rm_size += 1000000;
if( reg_degree >= rm_size ) return max_juint;
}
if( rm.overlap(lrgs(lidx).mask()) ) {
// Insert lidx into union LRG; returns TRUE if actually inserted
if( _ulr.insert(lidx) ) {
// Infinite-stack neighbors do not alter colorability, as they
// can always color to some other color.
if( !lrgs(lidx).mask().is_AllStack() ) {
// If this coalesce will make any new neighbor uncolorable,
// do not coalesce.
if( lrgs(lidx).just_lo_degree() )
return max_juint;
// Bump our degree
if( ++reg_degree >= rm_size )
return max_juint;
} // End of if not infinite-stack neighbor
} // End of if actually inserted
} // End of if live range overlaps
} // End of else collect interferences for 1 node
} // End of while forever, scan back for interferences
return reg_degree;
}
//------------------------------verify-----------------------------------------
void PhaseIFG::verify( const PhaseChaitin *pc ) const {
// IFG is square, sorted and no need for Find
for( uint i = 0; i < _maxlrg; i++ ) {
assert(!((*_yanked)[i]) || !neighbor_cnt(i), "Is removed completely" );
IndexSet *set = &_adjs[i];
IndexSetIterator elements(set);
uint idx;
uint last = 0;
while ((idx = elements.next()) != 0) {
assert( idx != i, "Must have empty diagonal");
assert( pc->Find_const(idx) == idx, "Must not need Find" );
assert( _adjs[idx].member(i), "IFG not square" );
assert( !(*_yanked)[idx], "No yanked neighbors" );
assert( last < idx, "not sorted increasing");
last = idx;
}
assert( !lrgs(i)._degree_valid ||
effective_degree(i) == lrgs(i).degree(), "degree is valid but wrong" );
}
}
//------------------------------re_insert--------------------------------------
// Re-insert a yanked Node.
void PhaseIFG::re_insert( uint a ) {
assert( _is_square, "only on square" );
assert( _yanked->test(a), "" );
(*_yanked) >>= a;
IndexSetIterator elements(&_adjs[a]);
uint datum;
while ((datum = elements.next()) != 0) {
_adjs[datum].insert(a);
lrgs(datum).invalid_degree();
}
}
//------------------------------copy_copy--------------------------------------
// See if I can coalesce a series of multiple copies together. I need the
// final dest copy and the original src copy. They can be the same Node.
// Compute the compatible register masks.
bool PhaseConservativeCoalesce::copy_copy( Node *dst_copy, Node *src_copy, Block *b, uint bindex ) {
if( !dst_copy->is_SpillCopy() ) return false;
if( !src_copy->is_SpillCopy() ) return false;
Node *src_def = src_copy->in(src_copy->is_Copy());
uint lr1 = _phc.Find(dst_copy);
uint lr2 = _phc.Find(src_def );
// Same live ranges already?
if( lr1 == lr2 ) return false;
// Interfere?
if( _phc._ifg->test_edge_sq( lr1, lr2 ) ) return false;
// Not an oop->int cast; oop->oop, int->int, AND int->oop are OK.
if( !lrgs(lr1)._is_oop && lrgs(lr2)._is_oop ) // not an oop->int cast
return false;
// Coalescing between an aligned live range and a mis-aligned live range?
// No, no! Alignment changes how we count degree.
if( lrgs(lr1)._fat_proj != lrgs(lr2)._fat_proj )
return false;
// Sort; use smaller live-range number
Node *lr1_node = dst_copy;
Node *lr2_node = src_def;
if( lr1 > lr2 ) {
uint tmp = lr1; lr1 = lr2; lr2 = tmp;
lr1_node = src_def; lr2_node = dst_copy;
}
// Check for compatibility of the 2 live ranges by
// intersecting their allowed register sets.
RegMask rm = lrgs(lr1).mask();
rm.AND(lrgs(lr2).mask());
// Number of bits free
uint rm_size = rm.Size();
if (UseFPUForSpilling && rm.is_AllStack() ) {
// Don't coalesce when frequency difference is large
Block *dst_b = _phc._cfg._bbs[dst_copy->_idx];
Block *src_def_b = _phc._cfg._bbs[src_def->_idx];
if (src_def_b->_freq > 10*dst_b->_freq )
return false;
}
// If we can use any stack slot, then effective size is infinite
if( rm.is_AllStack() ) rm_size += 1000000;
// Incompatible masks, no way to coalesce
if( rm_size == 0 ) return false;
// Another early bail-out test is when we are double-coalescing and the
// 2 copies are separated by some control flow.
if( dst_copy != src_copy ) {
Block *src_b = _phc._cfg._bbs[src_copy->_idx];
Block *b2 = b;
while( b2 != src_b ) {
if( b2->num_preds() > 2 ){// Found merge-point
_phc._lost_opp_cflow_coalesce++;
// extra record_bias commented out because Chris believes it is not
// productive. Since we can record only 1 bias, we want to choose one
// that stands a chance of working and this one probably does not.
//record_bias( _phc._lrgs, lr1, lr2 );
return false; // To hard to find all interferences
}
b2 = _phc._cfg._bbs[b2->pred(1)->_idx];
}
}
// Union the two interference sets together into '_ulr'
uint reg_degree = _ulr.lrg_union( lr1, lr2, rm_size, _phc._ifg, rm );
if( reg_degree >= rm_size ) {
record_bias( _phc._ifg, lr1, lr2 );
return false;
}
// Now I need to compute all the interferences between dst_copy and
// src_copy. I'm not willing visit the entire interference graph, so
// I limit my search to things in dst_copy's block or in a straight
// line of previous blocks. I give up at merge points or when I get
// more interferences than my degree. I can stop when I find src_copy.
if( dst_copy != src_copy ) {
reg_degree = compute_separating_interferences(dst_copy, src_copy, b, bindex, rm, rm_size, reg_degree, lr1, lr2 );
if( reg_degree == max_juint ) {
record_bias( _phc._ifg, lr1, lr2 );
return false;
}
} // End of if dst_copy & src_copy are different
// ---- THE COMBINED LRG IS COLORABLE ----
// YEAH - Now coalesce this copy away
assert( lrgs(lr1).num_regs() == lrgs(lr2).num_regs(), "" );
IndexSet *n_lr1 = _phc._ifg->neighbors(lr1);
IndexSet *n_lr2 = _phc._ifg->neighbors(lr2);
// Update the interference graph
update_ifg(lr1, lr2, n_lr1, n_lr2);
_ulr.remove(lr1);
// Uncomment the following code to trace Coalescing in great detail.
//
//if (false) {
// tty->cr();
// tty->print_cr("#######################################");
// tty->print_cr("union %d and %d", lr1, lr2);
// n_lr1->dump();
// n_lr2->dump();
// tty->print_cr("resulting set is");
// _ulr.dump();
//}
// Replace n_lr1 with the new combined live range. _ulr will use
// n_lr1's old memory on the next iteration. n_lr2 is cleared to
// send its internal memory to the free list.
_ulr.swap(n_lr1);
_ulr.clear();
n_lr2->clear();
lrgs(lr1).set_degree( _phc._ifg->effective_degree(lr1) );
lrgs(lr2).set_degree( 0 );
// Join live ranges. Merge larger into smaller. Union lr2 into lr1 in the
// union-find tree
union_helper( lr1_node, lr2_node, lr1, lr2, src_def, dst_copy, src_copy, b, bindex );
// Combine register restrictions
lrgs(lr1).set_mask(rm);
lrgs(lr1).compute_set_mask_size();
lrgs(lr1)._cost += lrgs(lr2)._cost;
lrgs(lr1)._area += lrgs(lr2)._area;
// While its uncommon to successfully coalesce live ranges that started out
// being not-lo-degree, it can happen. In any case the combined coalesced
// live range better Simplify nicely.
lrgs(lr1)._was_lo = 1;
// kinda expensive to do all the time
//tty->print_cr("warning: slow verify happening");
//_phc._ifg->verify( &_phc );
return true;
}
//------------------------------union_helper-----------------------------------
void PhaseConservativeCoalesce::union_helper( Node *lr1_node, Node *lr2_node, uint lr1, uint lr2, Node *src_def, Node *dst_copy, Node *src_copy, Block *b, uint bindex ) {
// Join live ranges. Merge larger into smaller. Union lr2 into lr1 in the
// union-find tree
_phc.Union( lr1_node, lr2_node );
// Single-def live range ONLY if both live ranges are single-def.
// If both are single def, then src_def powers one live range
// and def_copy powers the other. After merging, src_def powers
// the combined live range.
lrgs(lr1)._def = (lrgs(lr1).is_multidef() ||
lrgs(lr2).is_multidef() )
? NodeSentinel : src_def;
lrgs(lr2)._def = NULL; // No def for lrg 2
lrgs(lr2).Clear(); // Force empty mask for LRG 2
//lrgs(lr2)._size = 0; // Live-range 2 goes dead
lrgs(lr1)._is_oop |= lrgs(lr2)._is_oop;
lrgs(lr2)._is_oop = 0; // In particular, not an oop for GC info
if (lrgs(lr1)._maxfreq < lrgs(lr2)._maxfreq)
lrgs(lr1)._maxfreq = lrgs(lr2)._maxfreq;
// Copy original value instead. Intermediate copies go dead, and
// the dst_copy becomes useless.
int didx = dst_copy->is_Copy();
dst_copy->set_req( didx, src_def );
// Add copy to free list
// _phc.free_spillcopy(b->_nodes[bindex]);
assert( b->_nodes[bindex] == dst_copy, "" );
dst_copy->replace_by( dst_copy->in(didx) );
dst_copy->set_req( didx, NULL);
b->_nodes.remove(bindex);
if( bindex < b->_ihrp_index ) b->_ihrp_index--;
if( bindex < b->_fhrp_index ) b->_fhrp_index--;
// Stretched lr1; add it to liveness of intermediate blocks
Block *b2 = _phc._cfg._bbs[src_copy->_idx];
while( b != b2 ) {
b = _phc._cfg._bbs[b->pred(1)->_idx];
_phc._live->live(b)->insert(lr1);
}
}
//------------------------------insert_copies----------------------------------
void PhaseAggressiveCoalesce::insert_copies( Matcher &matcher ) {
// We do LRGs compressing and fix a liveout data only here since the other
// place in Split() is guarded by the assert which we never hit.
_phc.compress_uf_map_for_nodes();
// Fix block's liveout data for compressed live ranges.
for(uint lrg = 1; lrg < _phc._maxlrg; lrg++ ) {
uint compressed_lrg = _phc.Find(lrg);
if( lrg != compressed_lrg ) {
for( uint bidx = 0; bidx < _phc._cfg._num_blocks; bidx++ ) {
IndexSet *liveout = _phc._live->live(_phc._cfg._blocks[bidx]);
if( liveout->member(lrg) ) {
liveout->remove(lrg);
liveout->insert(compressed_lrg);
}
}
}
}
// All new nodes added are actual copies to replace virtual copies.
// Nodes with index less than '_unique' are original, non-virtual Nodes.
_unique = C->unique();
for( uint i=0; i<_phc._cfg._num_blocks; i++ ) {
Block *b = _phc._cfg._blocks[i];
uint cnt = b->num_preds(); // Number of inputs to the Phi
for( uint l = 1; l<b->_nodes.size(); l++ ) {
Node *n = b->_nodes[l];
// Do not use removed-copies, use copied value instead
uint ncnt = n->req();
for( uint k = 1; k<ncnt; k++ ) {
Node *copy = n->in(k);
uint cidx = copy->is_Copy();
if( cidx ) {
Node *def = copy->in(cidx);
if( _phc.Find(copy) == _phc.Find(def) )
n->set_req(k,def);
}
}
// Remove any explicit copies that get coalesced.
uint cidx = n->is_Copy();
if( cidx ) {
Node *def = n->in(cidx);
if( _phc.Find(n) == _phc.Find(def) ) {
n->replace_by(def);
n->set_req(cidx,NULL);
b->_nodes.remove(l);
l--;
continue;
}
}
if( n->is_Phi() ) {
// Get the chosen name for the Phi
uint phi_name = _phc.Find( n );
// Ignore the pre-allocated specials
if( !phi_name ) continue;
// Check for mismatch inputs to Phi
for( uint j = 1; j<cnt; j++ ) {
Node *m = n->in(j);
uint src_name = _phc.Find(m);
if( src_name != phi_name ) {
Block *pred = _phc._cfg._bbs[b->pred(j)->_idx];
Node *copy;
assert(!m->is_Con() || m->is_Mach(), "all Con must be Mach");
// Rematerialize constants instead of copying them
if( m->is_Mach() && m->as_Mach()->is_Con() &&
m->as_Mach()->rematerialize() ) {
copy = m->clone();
// Insert the copy in the predecessor basic block
pred->add_inst(copy);
// Copy any flags as well
_phc.clone_projs( pred, pred->end_idx(), m, copy, _phc._maxlrg );
} else {
const RegMask *rm = C->matcher()->idealreg2spillmask[m->ideal_reg()];
copy = new (C) MachSpillCopyNode(m,*rm,*rm);
// Find a good place to insert. Kinda tricky, use a subroutine
insert_copy_with_overlap(pred,copy,phi_name,src_name);
}
// Insert the copy in the use-def chain
n->set_req( j, copy );
_phc._cfg._bbs.map( copy->_idx, pred );
// Extend ("register allocate") the names array for the copy.
_phc._names.extend( copy->_idx, phi_name );
} // End of if Phi names do not match
} // End of for all inputs to Phi
} else { // End of if Phi
// Now check for 2-address instructions
uint idx;
if( n->is_Mach() && (idx=n->as_Mach()->two_adr()) ) {
// Get the chosen name for the Node
uint name = _phc.Find( n );
assert( name, "no 2-address specials" );
// Check for name mis-match on the 2-address input
Node *m = n->in(idx);
if( _phc.Find(m) != name ) {
Node *copy;
assert(!m->is_Con() || m->is_Mach(), "all Con must be Mach");
// At this point it is unsafe to extend live ranges (6550579).
// Rematerialize only constants as we do for Phi above.
if( m->is_Mach() && m->as_Mach()->is_Con() &&
m->as_Mach()->rematerialize() ) {
copy = m->clone();
// Insert the copy in the basic block, just before us
b->_nodes.insert( l++, copy );
if( _phc.clone_projs( b, l, m, copy, _phc._maxlrg ) )
l++;
} else {
const RegMask *rm = C->matcher()->idealreg2spillmask[m->ideal_reg()];
copy = new (C) MachSpillCopyNode( m, *rm, *rm );
// Insert the copy in the basic block, just before us
b->_nodes.insert( l++, copy );
}
// Insert the copy in the use-def chain
n->set_req(idx, copy );
// Extend ("register allocate") the names array for the copy.
_phc._names.extend( copy->_idx, name );
_phc._cfg._bbs.map( copy->_idx, b );
}
} // End of is two-adr
// Insert a copy at a debug use for a lrg which has high frequency
if( b->_freq < OPTO_DEBUG_SPLIT_FREQ || b->is_uncommon(_phc._cfg._bbs) ) {
// Walk the debug inputs to the node and check for lrg freq
JVMState* jvms = n->jvms();
uint debug_start = jvms ? jvms->debug_start() : 999999;
uint debug_end = jvms ? jvms->debug_end() : 999999;
for(uint inpidx = debug_start; inpidx < debug_end; inpidx++) {
// Do not split monitors; they are only needed for debug table
// entries and need no code.
if( jvms->is_monitor_use(inpidx) ) continue;
Node *inp = n->in(inpidx);
uint nidx = _phc.n2lidx(inp);
LRG &lrg = lrgs(nidx);
// If this lrg has a high frequency use/def
if( lrg._maxfreq >= _phc.high_frequency_lrg() ) {
// If the live range is also live out of this block (like it
// would be for a fast/slow idiom), the normal spill mechanism
// does an excellent job. If it is not live out of this block
// (like it would be for debug info to uncommon trap) splitting
// the live range now allows a better allocation in the high
// frequency blocks.
// Build_IFG_virtual has converted the live sets to
// live-IN info, not live-OUT info.
uint k;
for( k=0; k < b->_num_succs; k++ )
if( _phc._live->live(b->_succs[k])->member( nidx ) )
break; // Live in to some successor block?
if( k < b->_num_succs )
continue; // Live out; do not pre-split
// Split the lrg at this use
const RegMask *rm = C->matcher()->idealreg2spillmask[inp->ideal_reg()];
Node *copy = new (C) MachSpillCopyNode( inp, *rm, *rm );
// Insert the copy in the use-def chain
n->set_req(inpidx, copy );
// Insert the copy in the basic block, just before us
b->_nodes.insert( l++, copy );
// Extend ("register allocate") the names array for the copy.
_phc.new_lrg( copy, _phc._maxlrg++ );
_phc._cfg._bbs.map( copy->_idx, b );
//tty->print_cr("Split a debug use in Aggressive Coalesce");
} // End of if high frequency use/def
} // End of for all debug inputs
} // End of if low frequency safepoint
} // End of if Phi
} // End of for all instructions
} // End of for all blocks
}
//------------------------------Compute_Effective_Degree-----------------------
// Compute effective degree in bulk
void PhaseIFG::Compute_Effective_Degree() {
assert( _is_square, "only on square" );
for( uint i = 0; i < _maxlrg; i++ )
lrgs(i).set_degree(effective_degree(i));
}
