inline void G1ParScanClosure::do_oop_nv(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); const InCSetState state = _g1->in_cset_state(obj); if (state.is_in_cset()) { // We're not going to even bother checking whether the object is // already forwarded or not, as this usually causes an immediate // stall. We'll try to prefetch the object (for write, given that // we might need to install the forwarding reference) and we'll // get back to it when pop it from the queue Prefetch::write(obj->mark_addr(), 0); Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); // slightly paranoid test; I'm trying to catch potential // problems before we go into push_on_queue to know where the // problem is coming from assert((obj == oopDesc::load_decode_heap_oop(p)) || (obj->is_forwarded() && obj->forwardee() == oopDesc::load_decode_heap_oop(p)), "p should still be pointing to obj or to its forwardee"); _par_scan_state->push_on_queue(p); } else { if (state.is_humongous()) { _g1->set_humongous_is_live(obj); } _par_scan_state->update_rs(_from, p, _worker_id); } } }
uint max_regions(InCSetState dest) const { switch (dest.value()) { case InCSetState::Young: return _max_survivor_regions; case InCSetState::Old: return REGIONS_UNLIMITED; default: assert(false, "Unknown dest state: " CSETSTATE_FORMAT, dest.value()); break; } // keep some compilers happy return 0; }
void G1ParScanThreadState::report_promotion_event(InCSetState const dest_state, oop const old, size_t word_sz, uint age, HeapWord * const obj_ptr, const AllocationContext_t context) const { G1PLAB* alloc_buf = _plab_allocator->alloc_buffer(dest_state, context); if (alloc_buf->contains(obj_ptr)) { _g1h->_gc_tracer_stw->report_promotion_in_new_plab_event(old->klass(), word_sz, age, dest_state.value() == InCSetState::Old, alloc_buf->word_sz()); } else { _g1h->_gc_tracer_stw->report_promotion_outside_plab_event(old->klass(), word_sz, age, dest_state.value() == InCSetState::Old); } }
void G1ParCopyClosure<barrier, do_mark_object, use_ext>::do_oop_work(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (oopDesc::is_null(heap_oop)) { return; } oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); assert(_worker_id == _par_scan_state->worker_id(), "sanity"); const InCSetState state = _g1->in_cset_state(obj); if (state.is_in_cset()) { oop forwardee; markOop m = obj->mark(); if (m->is_marked()) { forwardee = (oop) m->decode_pointer(); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } assert(forwardee != NULL, "forwardee should not be NULL"); oopDesc::encode_store_heap_oop(p, forwardee); if (do_mark_object != G1MarkNone && forwardee != obj) { // If the object is self-forwarded we don't need to explicitly // mark it, the evacuation failure protocol will do so. mark_forwarded_object(obj, forwardee); } if (barrier == G1BarrierKlass) { do_klass_barrier(p, forwardee); } } else { if (state.is_humongous()) { _g1->set_humongous_is_live(obj); } if (use_ext && state.is_ext()) { _par_scan_state->do_oop_ext(p); } // The object is not in collection set. If we're a root scanning // closure during an initial mark pause then attempt to mark the object. if (do_mark_object == G1MarkFromRoot) { mark_object(obj); } } }
InCSetState G1ParScanThreadState::next_state(InCSetState const state, markOop const m, uint& age) { if (state.is_young()) { age = !m->has_displaced_mark_helper() ? m->age() : m->displaced_mark_helper()->age(); if (age < _tenuring_threshold) { return state; } } return dest(state); }
inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); const InCSetState state = _g1->in_cset_state(obj); if (state.is_in_cset_or_humongous()) { Prefetch::write(obj->mark_addr(), 0); Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); // Place on the references queue _par_scan_state->push_on_queue(p); } else if (state.is_ext()) { _par_scan_state->do_oop_ext(p); } else { assert(!_g1->obj_in_cs(obj), "checking"); } } }
oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state, oop const old, markOop const old_mark) { const size_t word_sz = old->size(); HeapRegion* const from_region = _g1h->heap_region_containing_raw(old); // +1 to make the -1 indexes valid... const int young_index = from_region->young_index_in_cset()+1; assert( (from_region->is_young() && young_index > 0) || (!from_region->is_young() && young_index == 0), "invariant" ); const AllocationContext_t context = from_region->allocation_context(); uint age = 0; InCSetState dest_state = next_state(state, old_mark, age); HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_state, word_sz, context); // PLAB allocations should succeed most of the time, so we'll // normally check against NULL once and that's it. if (obj_ptr == NULL) { obj_ptr = _plab_allocator->allocate_direct_or_new_plab(dest_state, word_sz, context); if (obj_ptr == NULL) { obj_ptr = allocate_in_next_plab(state, &dest_state, word_sz, context); if (obj_ptr == NULL) { // This will either forward-to-self, or detect that someone else has // installed a forwarding pointer. return handle_evacuation_failure_par(old, old_mark); } } } assert(obj_ptr != NULL, "when we get here, allocation should have succeeded"); assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap"); #ifndef PRODUCT // Should this evacuation fail? if (_g1h->evacuation_should_fail()) { // Doing this after all the allocation attempts also tests the // undo_allocation() method too. _plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); return handle_evacuation_failure_par(old, old_mark); } #endif // !PRODUCT // We're going to allocate linearly, so might as well prefetch ahead. Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); const oop obj = oop(obj_ptr); const oop forward_ptr = old->forward_to_atomic(obj); if (forward_ptr == NULL) { Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); if (dest_state.is_young()) { if (age < markOopDesc::max_age) { age++; } if (old_mark->has_displaced_mark_helper()) { // In this case, we have to install the mark word first, // otherwise obj looks to be forwarded (the old mark word, // which contains the forward pointer, was copied) obj->set_mark(old_mark); markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); old_mark->set_displaced_mark_helper(new_mark); } else { obj->set_mark(old_mark->set_age(age)); } age_table()->add(age, word_sz); } else { obj->set_mark(old_mark); } if (G1StringDedup::is_enabled()) { const bool is_from_young = state.is_young(); const bool is_to_young = dest_state.is_young(); assert(is_from_young == _g1h->heap_region_containing_raw(old)->is_young(), "sanity"); assert(is_to_young == _g1h->heap_region_containing_raw(obj)->is_young(), "sanity"); G1StringDedup::enqueue_from_evacuation(is_from_young, is_to_young, _worker_id, obj); } size_t* const surv_young_words = surviving_young_words(); surv_young_words[young_index] += word_sz; if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { // We keep track of the next start index in the length field of // the to-space object. The actual length can be found in the // length field of the from-space object. arrayOop(obj)->set_length(0); oop* old_p = set_partial_array_mask(old); push_on_queue(old_p); } else { HeapRegion* const to_region = _g1h->heap_region_containing_raw(obj_ptr); _scanner.set_region(to_region); obj->oop_iterate_backwards(&_scanner); } return obj; } else { _plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); return forward_ptr; } }