void PSMarkSweepDecorator::adjust_pointers() {
// adjust all the interior pointers to point at the new locations of objects
// Used by MarkSweep::mark_sweep_phase3()
HeapWord* q = space()->bottom();
HeapWord* t = _end_of_live; // Established by "prepare_for_compaction".
assert(_first_dead <= _end_of_live, "Stands to reason, no?");
if (q < t && _first_dead > q &&
!oop(q)->is_gc_marked()) {
// we have a chunk of the space which hasn't moved and we've
// reinitialized the mark word during the previous pass, so we can't
// use is_gc_marked for the traversal.
HeapWord* end = _first_dead;
while (q < end) {
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::track_interior_pointers(oop(q)));
// point all the oops to the new location
size_t size = oop(q)->adjust_pointers();
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers());
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::validate_live_oop(oop(q), size));
q += size;
}
if (_first_dead == t) {
q = t;
} else {
// $$$ This is funky. Using this to read the previously written
// LiveRange. See also use below.
q = (HeapWord*)oop(_first_dead)->mark()->decode_pointer();
}
}
const intx interval = PrefetchScanIntervalInBytes;
debug_only(HeapWord* prev_q = NULL);
while (q < t) {
// prefetch beyond q
Prefetch::write(q, interval);
if (oop(q)->is_gc_marked()) {
// q is alive
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::track_interior_pointers(oop(q)));
// point all the oops to the new location
size_t size = oop(q)->adjust_pointers();
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers());
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::validate_live_oop(oop(q), size));
debug_only(prev_q = q);
q += size;
} else {
// q is not a live object, so its mark should point at the next
// live object
debug_only(prev_q = q);
q = (HeapWord*) oop(q)->mark()->decode_pointer();
assert(q > prev_q, "we should be moving forward through memory");
}
}
assert(q == t, "just checking");
}
void Space::adjust_pointers() {
// adjust all the interior pointers to point at the new locations of objects
// Used by MarkSweep::mark_sweep_phase3()
// First check to see if there is any work to be done.
if (used() == 0) {
return; // Nothing to do.
}
// Otherwise...
HeapWord* q = bottom();
HeapWord* t = end();
debug_only(HeapWord* prev_q = NULL);
while (q < t) {
if (oop(q)->is_gc_marked()) {
// q is alive
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::track_interior_pointers(oop(q)));
// point all the oops to the new location
size_t size = oop(q)->adjust_pointers();
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::check_interior_pointers());
debug_only(prev_q = q);
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::validate_live_oop(oop(q), size));
q += size;
} else {
// q is not a live object. But we're not in a compactible space,
// So we don't have live ranges.
debug_only(prev_q = q);
q += block_size(q);
assert(q > prev_q, "we should be moving forward through memory");
}
}
assert(q == t, "just checking");
}
HeapWord* CompactibleSpace::forward(oop q, size_t size,
CompactPoint* cp, HeapWord* compact_top) {
// q is alive
// First check if we should switch compaction space
assert(this == cp->space, "'this' should be current compaction space.");
size_t compaction_max_size = pointer_delta(end(), compact_top);
while (size > compaction_max_size) {
// switch to next compaction space
cp->space->set_compaction_top(compact_top);
cp->space = cp->space->next_compaction_space();
if (cp->space == NULL) {
cp->gen = GenCollectedHeap::heap()->prev_gen(cp->gen);
assert(cp->gen != NULL, "compaction must succeed");
cp->space = cp->gen->first_compaction_space();
assert(cp->space != NULL, "generation must have a first compaction space");
}
compact_top = cp->space->bottom();
cp->space->set_compaction_top(compact_top);
cp->threshold = cp->space->initialize_threshold();
compaction_max_size = pointer_delta(cp->space->end(), compact_top);
}
// store the forwarding pointer into the mark word
if ((HeapWord*)q != compact_top) {
q->forward_to(oop(compact_top));
assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
} else {
// if the object isn't moving we can just set the mark to the default
// mark and handle it specially later on.
q->init_mark();
assert(q->forwardee() == NULL, "should be forwarded to NULL");
}
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::register_live_oop(q, size));
compact_top += size;
// we need to update the offset table so that the beginnings of objects can be
// found during scavenge. Note that we are updating the offset table based on
// where the object will be once the compaction phase finishes.
if (compact_top > cp->threshold)
cp->threshold =
cp->space->cross_threshold(compact_top - size, compact_top);
return compact_top;
}
template <class T> inline void MarkSweep::adjust_pointer(T* p, bool isroot) {
T heap_oop = oopDesc::load_heap_oop(p);
if (!oopDesc::is_null(heap_oop)) {
oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
oop new_obj = oop(obj->mark()->decode_pointer());
assert(new_obj != NULL || // is forwarding ptr?
obj->mark() == markOopDesc::prototype() || // not gc marked?
(UseBiasedLocking && obj->mark()->has_bias_pattern()) ||
// not gc marked?
obj->is_shared(), // never forwarded?
"should be forwarded");
if (new_obj != NULL) {
assert(Universe::heap()->is_in_reserved(new_obj),
"should be in object space");
oopDesc::encode_store_heap_oop_not_null(p, new_obj);
}
}
VALIDATE_MARK_SWEEP_ONLY(track_adjusted_pointer(p, isroot));
}
void PSMarkSweepDecorator::precompact() {
// Reset our own compact top.
set_compaction_top(space()->bottom());
/* We allow some amount of garbage towards the bottom of the space, so
* we don't start compacting before there is a significant gain to be made.
* Occasionally, we want to ensure a full compaction, which is determined
* by the MarkSweepAlwaysCompactCount parameter. This is a significant
* performance improvement!
*/
bool skip_dead = ((PSMarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0);
size_t allowed_deadspace = 0;
if (skip_dead) {
const size_t ratio = allowed_dead_ratio();
allowed_deadspace = space()->capacity_in_words() * ratio / 100;
}
// Fetch the current destination decorator
PSMarkSweepDecorator* dest = destination_decorator();
ObjectStartArray* start_array = dest->start_array();
HeapWord* compact_top = dest->compaction_top();
HeapWord* compact_end = dest->space()->end();
HeapWord* q = space()->bottom();
HeapWord* t = space()->top();
HeapWord* end_of_live= q; /* One byte beyond the last byte of the last
live object. */
HeapWord* first_dead = space()->end(); /* The first dead object. */
LiveRange* liveRange = NULL; /* The current live range, recorded in the
first header of preceding free area. */
_first_dead = first_dead;
const intx interval = PrefetchScanIntervalInBytes;
while (q < t) {
assert(oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() ||
oop(q)->mark()->has_bias_pattern(),
"these are the only valid states during a mark sweep");
if (oop(q)->is_gc_marked()) {
/* prefetch beyond q */
Prefetch::write(q, interval);
size_t size = oop(q)->size();
size_t compaction_max_size = pointer_delta(compact_end, compact_top);
// This should only happen if a space in the young gen overflows the
// old gen. If that should happen, we null out the start_array, because
// the young spaces are not covered by one.
while(size > compaction_max_size) {
// First record the last compact_top
dest->set_compaction_top(compact_top);
// Advance to the next compaction decorator
advance_destination_decorator();
dest = destination_decorator();
// Update compaction info
start_array = dest->start_array();
compact_top = dest->compaction_top();
compact_end = dest->space()->end();
assert(compact_top == dest->space()->bottom(), "Advanced to space already in use");
assert(compact_end > compact_top, "Must always be space remaining");
compaction_max_size =
pointer_delta(compact_end, compact_top);
}
// store the forwarding pointer into the mark word
if (q != compact_top) {
oop(q)->forward_to(oop(compact_top));
assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark");
} else {
// if the object isn't moving we can just set the mark to the default
// mark and handle it specially later on.
oop(q)->init_mark();
assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL");
}
// Update object start array
if (start_array) {
start_array->allocate_block(compact_top);
}
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::register_live_oop(oop(q), size));
compact_top += size;
assert(compact_top <= dest->space()->end(),
"Exceeding space in destination");
q += size;
end_of_live = q;
} else {
/* run over all the contiguous dead objects */
HeapWord* end = q;
do {
/* prefetch beyond end */
Prefetch::write(end, interval);
end += oop(end)->size();
} while (end < t && (!oop(end)->is_gc_marked()));
/* see if we might want to pretend this object is alive so that
* we don't have to compact quite as often.
*/
if (allowed_deadspace > 0 && q == compact_top) {
size_t sz = pointer_delta(end, q);
if (insert_deadspace(allowed_deadspace, q, sz)) {
size_t compaction_max_size = pointer_delta(compact_end, compact_top);
// This should only happen if a space in the young gen overflows the
// old gen. If that should happen, we null out the start_array, because
// the young spaces are not covered by one.
while (sz > compaction_max_size) {
// First record the last compact_top
dest->set_compaction_top(compact_top);
// Advance to the next compaction decorator
advance_destination_decorator();
dest = destination_decorator();
// Update compaction info
start_array = dest->start_array();
compact_top = dest->compaction_top();
compact_end = dest->space()->end();
assert(compact_top == dest->space()->bottom(), "Advanced to space already in use");
assert(compact_end > compact_top, "Must always be space remaining");
compaction_max_size =
pointer_delta(compact_end, compact_top);
}
// store the forwarding pointer into the mark word
if (q != compact_top) {
oop(q)->forward_to(oop(compact_top));
assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark");
} else {
// if the object isn't moving we can just set the mark to the default
// mark and handle it specially later on.
oop(q)->init_mark();
assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL");
}
// Update object start array
if (start_array) {
start_array->allocate_block(compact_top);
}
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::register_live_oop(oop(q), sz));
compact_top += sz;
assert(compact_top <= dest->space()->end(),
"Exceeding space in destination");
q = end;
end_of_live = end;
continue;
}
}
/* for the previous LiveRange, record the end of the live objects. */
if (liveRange) {
liveRange->set_end(q);
}
/* record the current LiveRange object.
* liveRange->start() is overlaid on the mark word.
*/
liveRange = (LiveRange*)q;
liveRange->set_start(end);
liveRange->set_end(end);
/* see if this is the first dead region. */
if (q < first_dead) {
first_dead = q;
}
/* move on to the next object */
q = end;
}
}
assert(q == t, "just checking");
if (liveRange != NULL) {
liveRange->set_end(q);
}
_end_of_live = end_of_live;
if (end_of_live < first_dead) {
first_dead = end_of_live;
}
_first_dead = first_dead;
// Update compaction top
dest->set_compaction_top(compact_top);
}
void PSMarkSweepDecorator::compact(bool mangle_free_space ) {
// Copy all live objects to their new location
// Used by MarkSweep::mark_sweep_phase4()
HeapWord* q = space()->bottom();
HeapWord* const t = _end_of_live;
debug_only(HeapWord* prev_q = NULL);
if (q < t && _first_dead > q &&
!oop(q)->is_gc_marked()) {
#ifdef ASSERT
// we have a chunk of the space which hasn't moved and we've reinitialized the
// mark word during the previous pass, so we can't use is_gc_marked for the
// traversal.
HeapWord* const end = _first_dead;
while (q < end) {
size_t size = oop(q)->size();
assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)");
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::live_oop_moved_to(q, size, q));
debug_only(prev_q = q);
q += size;
}
#endif
if (_first_dead == t) {
q = t;
} else {
// $$$ Funky
q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer();
}
}
const intx scan_interval = PrefetchScanIntervalInBytes;
const intx copy_interval = PrefetchCopyIntervalInBytes;
while (q < t) {
if (!oop(q)->is_gc_marked()) {
// mark is pointer to next marked oop
debug_only(prev_q = q);
q = (HeapWord*) oop(q)->mark()->decode_pointer();
assert(q > prev_q, "we should be moving forward through memory");
} else {
// prefetch beyond q
Prefetch::read(q, scan_interval);
// size and destination
size_t size = oop(q)->size();
HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee();
// prefetch beyond compaction_top
Prefetch::write(compaction_top, copy_interval);
// copy object and reinit its mark
VALIDATE_MARK_SWEEP_ONLY(MarkSweep::live_oop_moved_to(q, size, compaction_top));
assert(q != compaction_top, "everything in this pass should be moving");
Copy::aligned_conjoint_words(q, compaction_top, size);
oop(compaction_top)->init_mark();
assert(oop(compaction_top)->klass() != NULL, "should have a class");
debug_only(prev_q = q);
q += size;
}
}
assert(compaction_top() >= space()->bottom() && compaction_top() <= space()->end(),
"should point inside space");
space()->set_top(compaction_top());
if (mangle_free_space) {
space()->mangle_unused_area();
}
}
