void WorkGangBarrierSync::enter() {
MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag);
if (should_reset()) {
// The should_reset() was set and we are the first worker to enter
// the sync barrier. We will zero the n_completed() count which
// effectively resets the barrier.
zero_completed();
set_should_reset(false);
}
inc_completed();
if (n_completed() == n_workers()) {
// At this point we would like to reset the barrier to be ready in
// case it is used again. However, we cannot set n_completed() to
// 0, even after the notify_all(), given that some other workers
// might still be waiting for n_completed() to become ==
// n_workers(). So, if we set n_completed() to 0, those workers
// will get stuck (as they will wake up, see that n_completed() !=
// n_workers() and go back to sleep). Instead, we raise the
// should_reset() flag and the barrier will be reset the first
// time a worker enters it again.
set_should_reset(true);
monitor()->notify_all();
} else {
while (n_completed() != n_workers()) {
monitor()->wait(/* no_safepoint_check */ true);
}
}
}
void G1RootProcessor::wait_until_all_strong_classes_discovered() {
assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading");
if ((uint)_n_workers_discovered_strong_classes != n_workers()) {
MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag);
while ((uint)_n_workers_discovered_strong_classes != n_workers()) {
_lock.wait(Mutex::_no_safepoint_check_flag, 0, false);
}
}
}
void WorkGangBarrierSync::enter() {
MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag);
inc_completed();
if (n_completed() == n_workers()) {
monitor()->notify_all();
}
else {
while (n_completed() != n_workers()) {
monitor()->wait(/* no_safepoint_check */ true);
}
}
}
void G1RemSet::prepare_for_oops_into_collection_set_do() {
cleanupHRRS();
_g1->set_refine_cte_cl_concurrency(false);
DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
dcqs.concatenate_logs();
guarantee( _cards_scanned == NULL, "invariant" );
_cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers(), mtGC);
for (uint i = 0; i < n_workers(); ++i) {
_cards_scanned[i] = 0;
}
_total_cards_scanned = 0;
}
HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
: G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
_cg1r(g1->concurrent_g1_refine()),
_par_traversal_in_progress(false), _new_refs(NULL),
_cards_scanned(NULL), _total_cards_scanned(0)
{
_seq_task = new SubTasksDone(NumSeqTasks);
guarantee(n_workers() > 0, "There should be some workers");
_new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers());
for (uint i = 0; i < n_workers(); i++) {
_new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true);
}
}
G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
: _g1(g1), _conc_refine_cards(0),
_ct_bs(ct_bs), _g1p(_g1->g1_policy()),
_cg1r(g1->concurrent_g1_refine()),
_cset_rs_update_cl(NULL),
_cards_scanned(NULL), _total_cards_scanned(0)
{
_seq_task = new SubTasksDone(NumSeqTasks);
guarantee(n_workers() > 0, "There should be some workers");
_cset_rs_update_cl = NEW_C_HEAP_ARRAY(OopsInHeapRegionClosure*, n_workers());
for (uint i = 0; i < n_workers(); i++) {
_cset_rs_update_cl[i] = NULL;
}
}
HRInto_G1RemSet::~HRInto_G1RemSet() {
delete _seq_task;
for (uint i = 0; i < n_workers(); i++) {
delete _new_refs[i];
}
FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs);
}
G1RemSet::~G1RemSet() {
delete _seq_task;
for (uint i = 0; i < n_workers(); i++) {
assert(_cset_rs_update_cl[i] == NULL, "it should be");
}
FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl);
}
void G1RemSet::cleanup_after_oops_into_collection_set_do() {
guarantee( _cards_scanned != NULL, "invariant" );
_total_cards_scanned = 0;
for (uint i = 0; i < n_workers(); ++i) {
_total_cards_scanned += _cards_scanned[i];
}
FREE_C_HEAP_ARRAY(size_t, _cards_scanned);
_cards_scanned = NULL;
// Cleanup after copy
_g1->set_refine_cte_cl_concurrency(true);
// Set all cards back to clean.
_g1->cleanUpCardTable();
DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set();
int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
if (_g1->evacuation_failed()) {
double restore_remembered_set_start = os::elapsedTime();
// Restore remembered sets for the regions pointing into the collection set.
// We just need to transfer the completed buffers from the DirtyCardQueueSet
// used to hold cards that contain references that point into the collection set
// to the DCQS used to hold the deferred RS updates.
_g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
_g1->g1_policy()->phase_times()->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0);
}
// Free any completed buffers in the DirtyCardQueueSet used to hold cards
// which contain references that point into the collection.
_g1->into_cset_dirty_card_queue_set().clear();
assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0,
"all buffers should be freed");
_g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers();
}
void G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* oc,
CodeBlobClosure* code_root_cl,
uint worker_i) {
#if CARD_REPEAT_HISTO
ct_freq_update_histo_and_reset();
#endif
// We cache the value of 'oc' closure into the appropriate slot in the
// _cset_rs_update_cl for this worker
assert(worker_i < n_workers(), "sanity");
_cset_rs_update_cl[worker_i] = oc;
// A DirtyCardQueue that is used to hold cards containing references
// that point into the collection set. This DCQ is associated with a
// special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
// circumstances (i.e. the pause successfully completes), these cards
// are just discarded (there's no need to update the RSets of regions
// that were in the collection set - after the pause these regions
// are wholly 'free' of live objects. In the event of an evacuation
// failure the cards/buffers in this queue set are passed to the
// DirtyCardQueueSet that is used to manage RSet updates
DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
updateRS(&into_cset_dcq, worker_i);
scanRS(oc, code_root_cl, worker_i);
// We now clear the cached values of _cset_rs_update_cl for this worker
_cset_rs_update_cl[worker_i] = NULL;
}
template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) {
for (size_t i = 0; i < n_workers(); i++) {
for (int j = 0; j < _new_refs[i]->length(); j++) {
T* p = (T*) _new_refs[i]->at(j);
cl->do_oop(p);
}
}
}
void G1RootProcessor::process_strong_roots(OopClosure* oops,
CLDClosure* clds,
CodeBlobClosure* blobs) {
process_java_roots(oops, clds, clds, NULL, blobs, NULL, 0);
process_vm_roots(oops, NULL, NULL, 0);
_process_strong_tasks->all_tasks_completed(n_workers());
}
void G1RootProcessor::worker_has_discovered_all_strong_classes() {
assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading");
uint new_value = (uint)Atomic::add(1, &_n_workers_discovered_strong_classes);
if (new_value == n_workers()) {
// This thread is last. Notify the others.
MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag);
_lock.notify_all();
}
}
void G1RemSet::prepare_for_oops_into_collection_set_do() {
#if G1_REM_SET_LOGGING
PrintRSClosure cl;
_g1->collection_set_iterate(&cl);
#endif
cleanupHRRS();
ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
_g1->set_refine_cte_cl_concurrency(false);
DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
dcqs.concatenate_logs();
if (ParallelGCThreads > 0) {
_seq_task->set_n_threads((int)n_workers());
}
guarantee( _cards_scanned == NULL, "invariant" );
_cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers());
for (uint i = 0; i < n_workers(); ++i) {
_cards_scanned[i] = 0;
}
_total_cards_scanned = 0;
}
void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
guarantee( _cards_scanned != NULL, "invariant" );
_total_cards_scanned = 0;
for (uint i = 0; i < n_workers(); ++i)
_total_cards_scanned += _cards_scanned[i];
FREE_C_HEAP_ARRAY(size_t, _cards_scanned);
_cards_scanned = NULL;
// Cleanup after copy
#if G1_REM_SET_LOGGING
PrintRSClosure cl;
_g1->heap_region_iterate(&cl);
#endif
_g1->set_refine_cte_cl_concurrency(true);
cleanUpIteratorsClosure iterClosure;
_g1->collection_set_iterate(&iterClosure);
// Set all cards back to clean.
_g1->cleanUpCardTable();
if (ParallelGCThreads > 0) {
set_par_traversal(false);
}
if (_g1->evacuation_failed()) {
// Restore remembered sets for the regions pointing into
// the collection set.
if (G1DeferredRSUpdate) {
DirtyCardQueue dcq(&_g1->dirty_card_queue_set());
UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq);
new_refs_iterate(&deferred_update);
} else {
UpdateRSetOopsIntoCSImmediate immediate_update(_g1);
new_refs_iterate(&immediate_update);
}
}
for (uint i = 0; i < n_workers(); i++) {
_new_refs[i]->clear();
}
assert(!_par_traversal_in_progress, "Invariant between iterations.");
}
void G1RootProcessor::process_all_roots(OopClosure* oops,
CLDClosure* clds,
CodeBlobClosure* blobs) {
process_java_roots(oops, NULL, clds, clds, NULL, NULL, 0);
process_vm_roots(oops, oops, NULL, 0);
if (!_process_strong_tasks->is_task_claimed(G1RP_PS_CodeCache_oops_do)) {
CodeCache::blobs_do(blobs);
}
_process_strong_tasks->all_tasks_completed(n_workers());
}
void G1RemSet::cleanup_after_oops_into_collection_set_do() {
guarantee( _cards_scanned != NULL, "invariant" );
_total_cards_scanned = 0;
for (uint i = 0; i < n_workers(); ++i)
_total_cards_scanned += _cards_scanned[i];
FREE_C_HEAP_ARRAY(size_t, _cards_scanned);
_cards_scanned = NULL;
// Cleanup after copy
#if G1_REM_SET_LOGGING
PrintRSClosure cl;
_g1->heap_region_iterate(&cl);
#endif
_g1->set_refine_cte_cl_concurrency(true);
cleanUpIteratorsClosure iterClosure;
_g1->collection_set_iterate(&iterClosure);
// Set all cards back to clean.
_g1->cleanUpCardTable();
DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set();
int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
if (_g1->evacuation_failed()) {
// Restore remembered sets for the regions pointing into the collection set.
if (G1DeferredRSUpdate) {
// If deferred RS updates are enabled then we just need to transfer
// the completed buffers from (a) the DirtyCardQueueSet used to hold
// cards that contain references that point into the collection set
// to (b) the DCQS used to hold the deferred RS updates
_g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
} else {
CardTableModRefBS* bs = (CardTableModRefBS*)_g1->barrier_set();
UpdateRSetCardTableEntryIntoCSetClosure update_rs_cset_immediate(_g1, bs);
int n_completed_buffers = 0;
while (into_cset_dcqs.apply_closure_to_completed_buffer(&update_rs_cset_immediate,
0, 0, true)) {
n_completed_buffers++;
}
assert(n_completed_buffers == into_cset_n_buffers, "missed some buffers");
}
}
// Free any completed buffers in the DirtyCardQueueSet used to hold cards
// which contain references that point into the collection.
_g1->into_cset_dirty_card_queue_set().clear();
assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0,
"all buffers should be freed");
_g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers();
}
void G1RootProcessor::process_all_roots(OopClosure* oops,
CLDClosure* clds,
CodeBlobClosure* blobs,
bool process_string_table) {
AllRootsClosures closures(oops, clds);
process_java_roots(&closures, NULL, 0);
process_vm_roots(&closures, NULL, 0);
if (process_string_table) {
process_string_table_roots(&closures, NULL, 0);
}
process_code_cache_roots(blobs, NULL, 0);
_process_strong_tasks.all_tasks_completed(n_workers());
}
G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
: _g1(g1), _conc_refine_cards(0),
_ct_bs(ct_bs), _g1p(_g1->g1_policy()),
_cg1r(g1->concurrent_g1_refine()),
_cset_rs_update_cl(NULL),
_cards_scanned(NULL), _total_cards_scanned(0),
_prev_period_summary()
{
_seq_task = new SubTasksDone(NumSeqTasks);
_cset_rs_update_cl = NEW_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, n_workers(), mtGC);
for (uint i = 0; i < n_workers(); i++) {
_cset_rs_update_cl[i] = NULL;
}
if (G1SummarizeRSetStats) {
_prev_period_summary.initialize(this);
}
}
void G1RootProcessor::process_java_roots(G1RootClosures* closures,
G1GCPhaseTimes* phase_times,
uint worker_i) {
// Iterating over the CLDG and the Threads are done early to allow us to
// first process the strong CLDs and nmethods and then, after a barrier,
// let the thread process the weak CLDs and nmethods.
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CLDGRoots, worker_i);
if (!_process_strong_tasks.is_task_claimed(G1RP_PS_ClassLoaderDataGraph_oops_do)) {
ClassLoaderDataGraph::roots_cld_do(closures->strong_clds(), closures->weak_clds());
}
}
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ThreadRoots, worker_i);
bool is_par = n_workers() > 1;
Threads::possibly_parallel_oops_do(is_par,
closures->strong_oops(),
closures->strong_codeblobs());
}
}
void G1RootProcessor::process_java_roots(OopClosure* strong_roots,
CLDClosure* thread_stack_clds,
CLDClosure* strong_clds,
CLDClosure* weak_clds,
CodeBlobClosure* strong_code,
G1GCPhaseTimes* phase_times,
uint worker_i) {
assert(thread_stack_clds == NULL || weak_clds == NULL, "There is overlap between those, only one may be set");
// Iterating over the CLDG and the Threads are done early to allow us to
// first process the strong CLDs and nmethods and then, after a barrier,
// let the thread process the weak CLDs and nmethods.
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CLDGRoots, worker_i);
if (!_process_strong_tasks->is_task_claimed(G1RP_PS_ClassLoaderDataGraph_oops_do)) {
ClassLoaderDataGraph::roots_cld_do(strong_clds, weak_clds);
}
}
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ThreadRoots, worker_i);
bool is_par = n_workers() > 1;
Threads::possibly_parallel_oops_do(is_par, strong_roots, thread_stack_clds, strong_code);
}
}
void G1RootProcessor::evacuate_roots(G1EvacuationRootClosures* closures, uint worker_i) {
double ext_roots_start = os::elapsedTime();
G1GCPhaseTimes* phase_times = _g1h->g1_policy()->phase_times();
process_java_roots(closures, phase_times, worker_i);
// This is the point where this worker thread will not find more strong CLDs/nmethods.
// Report this so G1 can synchronize the strong and weak CLDs/nmethods processing.
if (closures->trace_metadata()) {
worker_has_discovered_all_strong_classes();
}
process_vm_roots(closures, phase_times, worker_i);
process_string_table_roots(closures, phase_times, worker_i);
{
// Now the CM ref_processor roots.
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CMRefRoots, worker_i);
if (!_process_strong_tasks.is_task_claimed(G1RP_PS_refProcessor_oops_do)) {
// We need to treat the discovered reference lists of the
// concurrent mark ref processor as roots and keep entries
// (which are added by the marking threads) on them live
// until they can be processed at the end of marking.
_g1h->ref_processor_cm()->weak_oops_do(closures->strong_oops());
}
}
if (closures->trace_metadata()) {
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WaitForStrongCLD, worker_i);
// Barrier to make sure all workers passed
// the strong CLD and strong nmethods phases.
wait_until_all_strong_classes_discovered();
}
// Now take the complement of the strong CLDs.
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WeakCLDRoots, worker_i);
assert(closures->second_pass_weak_clds() != NULL, "Should be non-null if we are tracing metadata.");
ClassLoaderDataGraph::roots_cld_do(NULL, closures->second_pass_weak_clds());
} else {
phase_times->record_time_secs(G1GCPhaseTimes::WaitForStrongCLD, worker_i, 0.0);
phase_times->record_time_secs(G1GCPhaseTimes::WeakCLDRoots, worker_i, 0.0);
assert(closures->second_pass_weak_clds() == NULL, "Should be null if not tracing metadata.");
}
// Finish up any enqueued closure apps (attributed as object copy time).
closures->flush();
double obj_copy_time_sec = closures->closure_app_seconds();
phase_times->record_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, obj_copy_time_sec);
double ext_root_time_sec = os::elapsedTime() - ext_roots_start - obj_copy_time_sec;
phase_times->record_time_secs(G1GCPhaseTimes::ExtRootScan, worker_i, ext_root_time_sec);
// During conc marking we have to filter the per-thread SATB buffers
// to make sure we remove any oops into the CSet (which will show up
// as implicitly live).
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SATBFiltering, worker_i);
if (!_process_strong_tasks.is_task_claimed(G1RP_PS_filter_satb_buffers) && _g1h->collector_state()->mark_in_progress()) {
JavaThread::satb_mark_queue_set().filter_thread_buffers();
}
}
_process_strong_tasks.all_tasks_completed(n_workers());
}
bool G1RemSet::refine_card(jbyte* card_ptr, uint worker_i,
bool check_for_refs_into_cset) {
assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)),
err_msg("Card at "PTR_FORMAT" index "SIZE_FORMAT" representing heap at "PTR_FORMAT" (%u) must be in committed heap",
p2i(card_ptr),
_ct_bs->index_for(_ct_bs->addr_for(card_ptr)),
_ct_bs->addr_for(card_ptr),
_g1->addr_to_region(_ct_bs->addr_for(card_ptr))));
// If the card is no longer dirty, nothing to do.
if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
// No need to return that this card contains refs that point
// into the collection set.
return false;
}
// Construct the region representing the card.
HeapWord* start = _ct_bs->addr_for(card_ptr);
// And find the region containing it.
HeapRegion* r = _g1->heap_region_containing(start);
// Why do we have to check here whether a card is on a young region,
// given that we dirty young regions and, as a result, the
// post-barrier is supposed to filter them out and never to enqueue
// them? When we allocate a new region as the "allocation region" we
// actually dirty its cards after we release the lock, since card
// dirtying while holding the lock was a performance bottleneck. So,
// as a result, it is possible for other threads to actually
// allocate objects in the region (after the acquire the lock)
// before all the cards on the region are dirtied. This is unlikely,
// and it doesn't happen often, but it can happen. So, the extra
// check below filters out those cards.
if (r->is_young()) {
return false;
}
// While we are processing RSet buffers during the collection, we
// actually don't want to scan any cards on the collection set,
// since we don't want to update remembered sets with entries that
// point into the collection set, given that live objects from the
// collection set are about to move and such entries will be stale
// very soon. This change also deals with a reliability issue which
// involves scanning a card in the collection set and coming across
// an array that was being chunked and looking malformed. Note,
// however, that if evacuation fails, we have to scan any objects
// that were not moved and create any missing entries.
if (r->in_collection_set()) {
return false;
}
// The result from the hot card cache insert call is either:
// * pointer to the current card
// (implying that the current card is not 'hot'),
// * null
// (meaning we had inserted the card ptr into the "hot" card cache,
// which had some headroom),
// * a pointer to a "hot" card that was evicted from the "hot" cache.
//
G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
if (hot_card_cache->use_cache()) {
assert(!check_for_refs_into_cset, "sanity");
assert(!SafepointSynchronize::is_at_safepoint(), "sanity");
card_ptr = hot_card_cache->insert(card_ptr);
if (card_ptr == NULL) {
// There was no eviction. Nothing to do.
return false;
}
start = _ct_bs->addr_for(card_ptr);
r = _g1->heap_region_containing(start);
// Checking whether the region we got back from the cache
// is young here is inappropriate. The region could have been
// freed, reallocated and tagged as young while in the cache.
// Hence we could see its young type change at any time.
}
// Don't use addr_for(card_ptr + 1) which can ask for
// a card beyond the heap. This is not safe without a perm
// gen at the upper end of the heap.
HeapWord* end = start + CardTableModRefBS::card_size_in_words;
MemRegion dirtyRegion(start, end);
#if CARD_REPEAT_HISTO
init_ct_freq_table(_g1->max_capacity());
ct_freq_note_card(_ct_bs->index_for(start));
#endif
G1ParPushHeapRSClosure* oops_in_heap_closure = NULL;
if (check_for_refs_into_cset) {
// ConcurrentG1RefineThreads have worker numbers larger than what
// _cset_rs_update_cl[] is set up to handle. But those threads should
// only be active outside of a collection which means that when they
// reach here they should have check_for_refs_into_cset == false.
assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
oops_in_heap_closure = _cset_rs_update_cl[worker_i];
}
G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
_g1->g1_rem_set(),
oops_in_heap_closure,
check_for_refs_into_cset,
worker_i);
update_rs_oop_cl.set_from(r);
G1TriggerClosure trigger_cl;
FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
(check_for_refs_into_cset ?
(OopClosure*)&mux :
(OopClosure*)&update_rs_oop_cl));
// The region for the current card may be a young region. The
// current card may have been a card that was evicted from the
// card cache. When the card was inserted into the cache, we had
// determined that its region was non-young. While in the cache,
// the region may have been freed during a cleanup pause, reallocated
// and tagged as young.
//
// We wish to filter out cards for such a region but the current
// thread, if we're running concurrently, may "see" the young type
// change at any time (so an earlier "is_young" check may pass or
// fail arbitrarily). We tell the iteration code to perform this
// filtering when it has been determined that there has been an actual
// allocation in this region and making it safe to check the young type.
bool filter_young = true;
HeapWord* stop_point =
r->oops_on_card_seq_iterate_careful(dirtyRegion,
&filter_then_update_rs_oop_cl,
filter_young,
card_ptr);
// If stop_point is non-null, then we encountered an unallocated region
// (perhaps the unfilled portion of a TLAB.) For now, we'll dirty the
// card and re-enqueue: if we put off the card until a GC pause, then the
// unallocated portion will be filled in. Alternatively, we might try
// the full complexity of the technique used in "regular" precleaning.
if (stop_point != NULL) {
// The card might have gotten re-dirtied and re-enqueued while we
// worked. (In fact, it's pretty likely.)
if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
*card_ptr = CardTableModRefBS::dirty_card_val();
MutexLockerEx x(Shared_DirtyCardQ_lock,
Mutex::_no_safepoint_check_flag);
DirtyCardQueue* sdcq =
JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
sdcq->enqueue(card_ptr);
}
} else {
_conc_refine_cards++;
}
// This gets set to true if the card being refined has
// references that point into the collection set.
bool has_refs_into_cset = trigger_cl.triggered();
// We should only be detecting that the card contains references
// that point into the collection set if the current thread is
// a GC worker thread.
assert(!has_refs_into_cset || SafepointSynchronize::is_at_safepoint(),
"invalid result at non safepoint");
return has_refs_into_cset;
}
void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
int worker_i) {
#if CARD_REPEAT_HISTO
ct_freq_update_histo_and_reset();
#endif
if (worker_i == 0) {
_cg1r->clear_and_record_card_counts();
}
// Make this into a command-line flag...
if (G1RSCountHisto && (ParallelGCThreads == 0 || worker_i == 0)) {
CountRSSizeClosure count_cl;
_g1->heap_region_iterate(&count_cl);
gclog_or_tty->print_cr("Avg of %d RS counts is %f, max is %d, "
"max region is " PTR_FORMAT,
count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(),
count_cl.mx(), count_cl.mxr());
count_cl.print_histo();
}
// We cache the value of 'oc' closure into the appropriate slot in the
// _cset_rs_update_cl for this worker
assert(worker_i < (int)n_workers(), "sanity");
_cset_rs_update_cl[worker_i] = oc;
// A DirtyCardQueue that is used to hold cards containing references
// that point into the collection set. This DCQ is associated with a
// special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
// circumstances (i.e. the pause successfully completes), these cards
// are just discarded (there's no need to update the RSets of regions
// that were in the collection set - after the pause these regions
// are wholly 'free' of live objects. In the event of an evacuation
// failure the cards/buffers in this queue set are:
// * passed to the DirtyCardQueueSet that is used to manage deferred
// RSet updates, or
// * scanned for references that point into the collection set
// and the RSet of the corresponding region in the collection set
// is updated immediately.
DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
assert((ParallelGCThreads > 0) || worker_i == 0, "invariant");
// The two flags below were introduced temporarily to serialize
// the updating and scanning of remembered sets. There are some
// race conditions when these two operations are done in parallel
// and they are causing failures. When we resolve said race
// conditions, we'll revert back to parallel remembered set
// updating and scanning. See CRs 6677707 and 6677708.
if (G1UseParallelRSetUpdating || (worker_i == 0)) {
updateRS(&into_cset_dcq, worker_i);
} else {
_g1p->record_update_rs_processed_buffers(worker_i, 0.0);
_g1p->record_update_rs_time(worker_i, 0.0);
}
if (G1UseParallelRSetScanning || (worker_i == 0)) {
scanRS(oc, worker_i);
} else {
_g1p->record_scan_rs_time(worker_i, 0.0);
}
// We now clear the cached values of _cset_rs_update_cl for this worker
_cset_rs_update_cl[worker_i] = NULL;
}
void G1RootProcessor::evacuate_roots(OopClosure* scan_non_heap_roots,
OopClosure* scan_non_heap_weak_roots,
CLDClosure* scan_strong_clds,
CLDClosure* scan_weak_clds,
bool trace_metadata,
uint worker_i) {
// First scan the shared roots.
double ext_roots_start = os::elapsedTime();
G1GCPhaseTimes* phase_times = _g1h->g1_policy()->phase_times();
BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
BufferingOopClosure buf_scan_non_heap_weak_roots(scan_non_heap_weak_roots);
OopClosure* const weak_roots = &buf_scan_non_heap_weak_roots;
OopClosure* const strong_roots = &buf_scan_non_heap_roots;
// CodeBlobClosures are not interoperable with BufferingOopClosures
G1CodeBlobClosure root_code_blobs(scan_non_heap_roots);
process_java_roots(strong_roots,
trace_metadata ? scan_strong_clds : NULL,
scan_strong_clds,
trace_metadata ? NULL : scan_weak_clds,
&root_code_blobs,
phase_times,
worker_i);
// This is the point where this worker thread will not find more strong CLDs/nmethods.
// Report this so G1 can synchronize the strong and weak CLDs/nmethods processing.
if (trace_metadata) {
worker_has_discovered_all_strong_classes();
}
process_vm_roots(strong_roots, weak_roots, phase_times, worker_i);
{
// Now the CM ref_processor roots.
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CMRefRoots, worker_i);
if (!_process_strong_tasks->is_task_claimed(G1RP_PS_refProcessor_oops_do)) {
// We need to treat the discovered reference lists of the
// concurrent mark ref processor as roots and keep entries
// (which are added by the marking threads) on them live
// until they can be processed at the end of marking.
_g1h->ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots);
}
}
if (trace_metadata) {
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WaitForStrongCLD, worker_i);
// Barrier to make sure all workers passed
// the strong CLD and strong nmethods phases.
wait_until_all_strong_classes_discovered();
}
// Now take the complement of the strong CLDs.
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WeakCLDRoots, worker_i);
ClassLoaderDataGraph::roots_cld_do(NULL, scan_weak_clds);
} else {
phase_times->record_time_secs(G1GCPhaseTimes::WaitForStrongCLD, worker_i, 0.0);
phase_times->record_time_secs(G1GCPhaseTimes::WeakCLDRoots, worker_i, 0.0);
}
// Finish up any enqueued closure apps (attributed as object copy time).
buf_scan_non_heap_roots.done();
buf_scan_non_heap_weak_roots.done();
double obj_copy_time_sec = buf_scan_non_heap_roots.closure_app_seconds()
+ buf_scan_non_heap_weak_roots.closure_app_seconds();
phase_times->record_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, obj_copy_time_sec);
double ext_root_time_sec = os::elapsedTime() - ext_roots_start - obj_copy_time_sec;
phase_times->record_time_secs(G1GCPhaseTimes::ExtRootScan, worker_i, ext_root_time_sec);
// During conc marking we have to filter the per-thread SATB buffers
// to make sure we remove any oops into the CSet (which will show up
// as implicitly live).
{
G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SATBFiltering, worker_i);
if (!_process_strong_tasks->is_task_claimed(G1RP_PS_filter_satb_buffers) && _g1h->collector_state()->mark_in_progress()) {
JavaThread::satb_mark_queue_set().filter_thread_buffers();
}
}
_process_strong_tasks->all_tasks_completed(n_workers());
}
int ChainInfo::
dead_workers()
{
auto workers = chain_.read ()->workers().read ();
return workers->workers().size() - workers->n_workers();
}
