/* Finalizable objs falls back to objs with fin when resurrection fallback happens */
static void finalizable_objs_fallback(GC *gc)
{
Finref_Metadata *metadata = gc->finref_metadata;
Pool *finalizable_obj_pool = metadata->finalizable_obj_pool;
Pool *obj_with_fin_pool = metadata->obj_with_fin_pool;
Vector_Block *obj_with_fin_block = finref_get_free_block(gc);
Vector_Block *block = pool_get_entry(finalizable_obj_pool);
while(block){
POINTER_SIZE_INT *iter = vector_block_iterator_init(block);
for(; !vector_block_iterator_end(block, iter); iter = vector_block_iterator_advance(block, iter)){
REF *p_ref = (REF*)iter;
Partial_Reveal_Object *p_obj = read_slot(p_ref);
assert(p_obj);
/* Perhaps obj has been resurrected by previous resurrections. If the fin-obj was resurrected, we need put it back to obj_with_fin pool.
For minor collection, the resurrected obj was forwarded, so we need use the new copy.*/
if(!gc_obj_is_dead(gc, p_obj) && obj_belongs_to_nos(p_obj)){
/* Even in NOS, not all live objects are forwarded due to the partial-forward algortihm */
if(!NOS_PARTIAL_FORWARD || fspace_obj_to_be_forwarded(p_obj)){
write_slot(p_ref , obj_get_fw_in_oi(p_obj));
p_obj = read_slot(p_ref);
}
}
/* Perhaps obj_with_fin_block has been allocated with a new free block if it is full */
obj_with_fin_block = gc_add_finalizer(gc, obj_with_fin_block, p_obj);
}
block = pool_get_entry(finalizable_obj_pool);
}
pool_put_entry(obj_with_fin_pool, obj_with_fin_block);
metadata->pending_finalizers = FALSE;
}
static FORCE_INLINE void scan_slot(Collector *collector, REF *p_ref)
{
Partial_Reveal_Object *p_obj = read_slot(p_ref);
if( p_obj == NULL) return;
/* the slot can be in tspace or fspace, we don't care. In gen mode,
we care only if the reference in the slot is pointing to nos */
if (obj_belongs_to_nos(p_obj))
collector_tracestack_push(collector, p_ref);
return;
}
static inline void fallback_update_fw_ref(REF *p_ref)
{
assert(collect_is_fallback());
Partial_Reveal_Object *p_obj = read_slot(p_ref);
if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){
assert(!obj_is_marked_in_vt(p_obj));
assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj)));
p_obj = obj_get_fw_in_oi(p_obj);
assert(p_obj);
write_slot(p_ref, p_obj);
}
}
void verifier_trace_objsets(Heap_Verifier* heap_verifier, Pool* obj_set_pool)
{
Heap_Verifier_Metadata* verifier_metadata = heap_verifier->heap_verifier_metadata;
GC_Verifier* gc_verifier = heap_verifier->gc_verifier;
gc_verifier->trace_stack = verifier_free_task_pool_get_entry(verifier_metadata->free_task_pool);
pool_iterator_init(obj_set_pool);
Vector_Block* obj_set = pool_iterator_next(obj_set_pool);
/* first step: copy all root objects to trace tasks. */
while(obj_set){
POINTER_SIZE_INT* iter = vector_block_iterator_init(obj_set);
while(!vector_block_iterator_end(obj_set,iter)){
Partial_Reveal_Object* p_obj = read_slot((REF*)iter);
iter = vector_block_iterator_advance(obj_set,iter);
/*p_obj can be NULL , When GC happened, the obj in Finalize objs list will be clear.*/
//assert(p_obj != NULL);
if(p_obj == NULL) continue;
if(heap_verifier->gc_is_gen_mode && heap_verifier->is_before_gc && !obj_belongs_to_nos(p_obj)) continue;
verifier_tracestack_push(p_obj, gc_verifier->trace_stack);
}
obj_set = pool_iterator_next(obj_set_pool);
}
/* put back the last trace_stack task */
pool_put_entry(verifier_metadata->mark_task_pool, gc_verifier->trace_stack);
/* second step: iterate over the trace tasks and forward objects */
gc_verifier->trace_stack = verifier_free_task_pool_get_entry(verifier_metadata->free_task_pool);
Vector_Block* trace_task = pool_get_entry(verifier_metadata->mark_task_pool);
while(trace_task){
POINTER_SIZE_INT* iter = vector_block_iterator_init(trace_task);
while(!vector_block_iterator_end(trace_task,iter)){
Partial_Reveal_Object* p_obj = (Partial_Reveal_Object* )*iter;
iter = vector_block_iterator_advance(trace_task,iter);
trace_object(heap_verifier, p_obj);
}
vector_stack_clear(trace_task);
pool_put_entry(verifier_metadata->free_task_pool, trace_task);
trace_task = pool_get_entry(verifier_metadata->mark_task_pool);
}
vector_stack_clear(gc_verifier->trace_stack);
pool_put_entry(verifier_metadata->free_task_pool, gc_verifier->trace_stack);
gc_verifier->trace_stack = NULL;
}
static FORCE_INLINE void scan_object(Heap_Verifier* heap_verifier, Partial_Reveal_Object *p_obj)
{
GC_Verifier* gc_verifier = heap_verifier->gc_verifier;
#if !defined(USE_UNIQUE_MARK_SWEEP_GC) && !defined(USE_UNIQUE_MOVE_COMPACT_GC)
if(gc_verifier->is_before_fallback_collection) {
if(obj_belongs_to_nos(p_obj) && obj_is_fw_in_oi(p_obj)){
assert(obj_get_vt(p_obj) == obj_get_vt(obj_get_fw_in_oi(p_obj)));
p_obj = obj_get_fw_in_oi(p_obj);
assert(p_obj);
}
}
#endif
if(!obj_mark_in_vt(p_obj)) return;
if( !major_is_marksweep() && p_obj >= los_boundary ){
Block_Header* block = GC_BLOCK_HEADER(p_obj);
if( heap_verifier->is_before_gc) block->num_live_objs++;
/* we can't set block->num_live_objs = 0 if !is_before_gc, because the some blocks may be freed hence not
visited after GC. So we should reset it in GC space reset functions. */
}
verify_object_header(p_obj, heap_verifier);
verifier_update_verify_info(p_obj, heap_verifier);
/*FIXME: */
if (!object_has_ref_field(p_obj)) return;
REF* p_ref;
if (object_is_array(p_obj)) {
Partial_Reveal_Array* array = (Partial_Reveal_Array*)p_obj;
unsigned int array_length = array->array_len;
p_ref = (REF*)((POINTER_SIZE_INT)array + (int)array_first_element_offset(array));
for (unsigned int i = 0; i < array_length; i++) {
scan_slot(heap_verifier, p_ref+i);
}
}else{
unsigned int num_refs = object_ref_field_num(p_obj);
int* ref_iterator = object_ref_iterator_init(p_obj);
for(unsigned int i=0; i<num_refs; i++){
p_ref = object_ref_iterator_get(ref_iterator+i, p_obj);
scan_slot(heap_verifier, p_ref);
}
#ifndef BUILD_IN_REFERENT
WeakReferenceType type = special_reference_type(p_obj);
if(type == SOFT_REFERENCE && verifier_collect_is_minor(gc_verifier)){
p_ref = obj_get_referent_field(p_obj);
scan_slot(heap_verifier, p_ref);
}
#endif
}
return;
}
static FORCE_INLINE void forward_object(Collector* collector, REF *p_ref)
{
GC* gc = collector->gc;
Partial_Reveal_Object *p_obj = read_slot(p_ref);
if(obj_belongs_to_tospace(p_obj)) return;
if(!obj_belongs_to_nos(p_obj)){
if(obj_mark_in_oi(p_obj)){
#ifdef GC_GEN_STATS
if(gc_profile){
GC_Gen_Collector_Stats* stats = (GC_Gen_Collector_Stats*)collector->stats;
gc_gen_collector_update_marked_nonnos_obj_stats_minor(stats);
}
#endif
scan_object(collector, p_obj);
}
return;
}
Partial_Reveal_Object* p_target_obj = NULL;
/* Fastpath: object has already been forwarded, update the ref slot */
if(obj_is_fw_in_oi(p_obj)) {
p_target_obj = obj_get_fw_in_oi(p_obj);
assert(p_target_obj);
write_slot(p_ref, p_target_obj);
return;
}
/* following is the logic for forwarding */
p_target_obj = collector_forward_object(collector, p_obj);
/* if p_target_obj is NULL, it is forwarded by other thread.
We can implement the collector_forward_object() so that the forwarding pointer
is set in the atomic instruction, which requires to roll back the mos_alloced
space. That is easy for thread local block allocation cancellation. */
if( p_target_obj == NULL ){
if(collector->result == FALSE ){
/* failed to forward, let's get back to controller. */
vector_stack_clear(collector->trace_stack);
return;
}
p_target_obj = obj_get_fw_in_oi(p_obj);
assert(p_target_obj);
write_slot(p_ref, p_target_obj);
return;
}
/* otherwise, we successfully forwarded */
#ifdef GC_GEN_STATS
if(gc_profile){
GC_Gen_Collector_Stats* stats = (GC_Gen_Collector_Stats*)collector->stats;
gc_gen_collector_update_marked_nos_obj_stats_minor(stats);
gc_gen_collector_update_moved_nos_obj_stats_minor(stats, vm_object_size(p_obj));
}
#endif
write_slot(p_ref, p_target_obj);
scan_object(collector, p_target_obj);
return;
}
static FORCE_INLINE void forward_object(Collector *collector, REF *p_ref)
{
Space* space = collector->collect_space;
GC* gc = collector->gc;
Partial_Reveal_Object *p_obj = read_slot(p_ref);
/* p_obj can also be in tospace because this p_ref is a redundant one in mutator remset.
We don't rem p_ref because it was remembered in first time it's met.
FIXME:: the situation obj_belongs_to_tospace() should never be true if we
remember object rather than slot. Currently, mutator remembers objects, and
collector remembers slots. Although collectors remember slots, we are sure
there are no chances to have repetitive p_ref because an object is scanned only
when it is marked or forwarded atomically, hence only one collector has chance
to do the scanning. */
if(!obj_belongs_to_nos(p_obj) || obj_belongs_to_tospace(p_obj)) return;
Partial_Reveal_Object* p_target_obj = NULL;
Boolean to_rem_slot = FALSE;
/* Fastpath: object has already been forwarded, update the ref slot */
if(obj_is_fw_in_oi(p_obj)){
p_target_obj = obj_get_fw_in_oi(p_obj);
write_slot(p_ref, p_target_obj);
/* check if the target obj stays in NOS, and p_ref from MOS. If yes, rem p_ref. */
if(obj_belongs_to_tospace(p_target_obj))
if( !addr_belongs_to_nos(p_ref) && address_belongs_to_gc_heap(p_ref, gc))
collector_remset_add_entry(collector, ( Partial_Reveal_Object**) p_ref);
return;
}
/* following is the logic for forwarding */
p_target_obj = collector_forward_object(collector, p_obj);
/* if p_target_obj is NULL, it is forwarded by other thread.
Note: a race condition here, it might be forwarded by other, but not set the
forwarding pointer yet. We need spin here to get the forwarding pointer.
We can implement the collector_forward_object() so that the forwarding pointer
is set in the atomic instruction, which requires to roll back the mos_alloced
space. That is easy for thread local block allocation cancellation. */
if( p_target_obj == NULL ){
if(collector->result == FALSE ){
/* failed to forward, let's get back to controller. */
vector_stack_clear(collector->trace_stack);
return;
}
/* forwarded already*/
p_target_obj = obj_get_fw_in_oi(p_obj);
}else{ /* otherwise, we successfully forwarded */
#ifdef GC_GEN_STATS
if(gc_profile){
GC_Gen_Collector_Stats* stats = (GC_Gen_Collector_Stats*)collector->stats;
gc_gen_collector_update_marked_nos_obj_stats_minor(stats);
gc_gen_collector_update_moved_nos_obj_stats_minor(stats, vm_object_size(p_obj));
}
#endif
scan_object(collector, p_target_obj);
}
assert(p_target_obj);
write_slot(p_ref, p_target_obj);
/* check if the target obj stays in NOS, and p_ref from MOS. If yes, rem p_ref. */
if(obj_belongs_to_tospace(p_target_obj)){
if( !addr_belongs_to_nos(p_ref) && address_belongs_to_gc_heap(p_ref, gc))
collector_remset_add_entry(collector, ( Partial_Reveal_Object**) p_ref);
}
return;
}
static void collector_trace_rootsets(Collector* collector)
{
GC* gc = collector->gc;
GC_Metadata* metadata = gc->metadata;
#ifdef GC_GEN_STATS
GC_Gen_Collector_Stats* stats = (GC_Gen_Collector_Stats*)collector->stats;
#endif
unsigned int num_active_collectors = gc->num_active_collectors;
atomic_cas32( &num_finished_collectors, 0, num_active_collectors);
Space* space = collector->collect_space;
collector->trace_stack = free_task_pool_get_entry(metadata);
/* find root slots saved by 1. active mutators, 2. exited mutators, 3. last cycle collectors */
Vector_Block* root_set = pool_iterator_next(metadata->gc_rootset_pool);
/* first step: copy all root objects to trace tasks. */
TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: copy root objects to trace stack ......");
while(root_set){
POINTER_SIZE_INT* iter = vector_block_iterator_init(root_set);
while(!vector_block_iterator_end(root_set,iter)){
REF *p_ref = (REF *)*iter;
iter = vector_block_iterator_advance(root_set,iter);
if(!*p_ref) continue; /* root ref cann't be NULL, but remset can be */
Partial_Reveal_Object *p_obj = read_slot(p_ref);
#ifdef GC_GEN_STATS
gc_gen_collector_update_rootset_ref_num(stats);
#endif
if(obj_belongs_to_nos(p_obj)){
collector_tracestack_push(collector, p_ref);
}
}
root_set = pool_iterator_next(metadata->gc_rootset_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, collector->trace_stack);
/* second step: iterate over the trace tasks and forward objects */
collector->trace_stack = free_task_pool_get_entry(metadata);
TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish copying root objects to trace stack.");
TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: trace and forward objects ......");
retry:
Vector_Block* trace_task = pool_get_entry(metadata->mark_task_pool);
while(trace_task){
POINTER_SIZE_INT* iter = vector_block_iterator_init(trace_task);
while(!vector_block_iterator_end(trace_task,iter)){
REF *p_ref = (REF *)*iter;
iter = vector_block_iterator_advance(trace_task,iter);
assert(*p_ref); /* a task can't be NULL, it was checked before put into the task stack */
#ifdef PREFETCH_SUPPORTED
/* DO PREFETCH */
if( mark_prefetch ) {
if(!vector_block_iterator_end(trace_task, iter)) {
REF *pref= (REF*) *iter;
PREFETCH( read_slot(pref));
}
}
#endif
/* in sequential version, we only trace same object once, but we were using a local hashset for that,
which couldn't catch the repetition between multiple collectors. This is subject to more study. */
/* FIXME:: we should not let root_set empty during working, other may want to steal it.
degenerate my stack into root_set, and grab another stack */
/* a task has to belong to collected space, it was checked before put into the stack */
trace_object(collector, p_ref);
if(collector->result == FALSE) break; /* force return */
}
vector_stack_clear(trace_task);
pool_put_entry(metadata->free_task_pool, trace_task);
if(collector->result == FALSE){
gc_task_pool_clear(metadata->mark_task_pool);
break; /* force return */
}
trace_task = pool_get_entry(metadata->mark_task_pool);
}
atomic_inc32(&num_finished_collectors);
while(num_finished_collectors != num_active_collectors){
if( pool_is_empty(metadata->mark_task_pool)) continue;
/* we can't grab the task here, because of a race condition. If we grab the task,
and the pool is empty, other threads may fall to this barrier and then pass. */
atomic_dec32(&num_finished_collectors);
goto retry;
}
TRACE2("gc.process", "GC: collector["<<((POINTER_SIZE_INT)collector->thread_handle)<<"]: finish tracing and forwarding objects.");
/* now we are done, but each collector has a private stack that is empty */
trace_task = (Vector_Block*)collector->trace_stack;
vector_stack_clear(trace_task);
pool_put_entry(metadata->free_task_pool, trace_task);
collector->trace_stack = NULL;
return;
}
