/* In two cases mark-sweep needs fixing repointed refs:
* 1. ms with compaction
* 2. ms as a mos collection algorithm
*/
static inline void moving_mark_sweep_update_ref(GC *gc, REF *p_ref, Boolean double_fix)
{
/* There are only two kinds of p_ref being added into finref_repset_pool:
* 1. p_ref is in a vector block from one finref pool;
* 2. p_ref is a referent field.
* So if p_ref belongs to heap, it must be a referent field pointer.
* Objects except a tree root which are resurrected need not be recorded in finref_repset_pool.
*/
if(address_belongs_to_gc_heap((void*)p_ref, gc)){
unsigned int offset = get_gc_referent_offset();
Partial_Reveal_Object *p_old_ref = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_ref - offset);
if(obj_is_fw_in_oi(p_old_ref)){
Partial_Reveal_Object *p_new_ref = obj_get_fw_in_oi(p_old_ref);
/* Only major collection in MS Gen GC might need double_fix.
* Double fixing happens when both forwarding and compaction happen.
*/
if(double_fix && obj_is_fw_in_oi(p_new_ref)){
assert(major_is_marksweep());
p_new_ref = obj_get_fw_in_oi(p_new_ref);
assert(address_belongs_to_gc_heap(p_new_ref, gc));
}
p_ref = (REF*)((POINTER_SIZE_INT)p_new_ref + offset);
}
}
Partial_Reveal_Object *p_obj = read_slot(p_ref);
/* assert(obj_need_move(gc, p_obj));
* This assertion is commented out because it assert(!obj_is_dead(gc, p_obj)).
* When gc_fix_rootset is invoked, mark bit and alloc bit have been flipped in Mark-Sweep,
* so this assertion will fail.
* But for sure p_obj here must be an one needing moving.
*/
p_obj = obj_get_fw_in_oi(p_obj);
/* Only major collection in MS Gen GC might need double_fix.
* Double fixing happens when both forwarding and compaction happen.
*/
if(double_fix && obj_is_fw_in_oi(p_obj)){
assert(major_is_marksweep());
p_obj = obj_get_fw_in_oi(p_obj);
assert(address_belongs_to_gc_heap(p_obj, gc));
}
write_slot(p_ref, p_obj);
}
static FORCE_INLINE void scan_slot(Collector* marker, REF *p_ref)
{
Partial_Reveal_Object *p_obj = read_slot(p_ref);
if( p_obj == NULL) return;
assert(address_belongs_to_gc_heap(p_obj, marker->gc));
if(obj_mark_gray_in_table(p_obj)){
assert(p_obj);
collector_tracestack_push((Collector*)marker, p_obj);
}
}
/* Move compaction needs special treament when updating referent field */
static inline void move_compaction_update_ref(GC *gc, REF *p_ref)
{
/* There are only two kinds of p_ref being added into finref_repset_pool:
* 1. p_ref is in a vector block from one finref pool;
* 2. p_ref is a referent field.
* So if p_ref belongs to heap, it must be a referent field pointer.
* Objects except a tree root which are resurrected need not be recorded in finref_repset_pool.
*/
if(address_belongs_to_gc_heap(p_ref, gc) && (p_ref >= los_boundary)){
unsigned int offset = get_gc_referent_offset();
Partial_Reveal_Object *p_old_ref = (Partial_Reveal_Object*)((POINTER_SIZE_INT)p_ref - offset);
Partial_Reveal_Object *p_new_ref = obj_get_fw_in_table(p_old_ref);
p_ref = (REF*)((POINTER_SIZE_INT)p_new_ref + offset);
}
Partial_Reveal_Object *p_obj = read_slot(p_ref);
assert(space_of_addr(gc, p_obj)->move_object);
if(p_obj < los_boundary)
p_obj = obj_get_fw_in_oi(p_obj);
else
p_obj = obj_get_fw_in_table(p_obj);
write_slot(p_ref, p_obj);
}
void wspace_final_mark_scan_mostly_concurrent(Conclctor* marker)
{
GC *gc = marker->gc;
GC_Metadata *metadata = gc->metadata;
unsigned int num_dirtyset_slot = 0;
marker->trace_stack = free_task_pool_get_entry(metadata);
Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool);
/* first step: copy all root objects to mark tasks.*/
while(root_set){
POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set);
while(!vector_block_iterator_end(root_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(root_set,iter);
assert(p_obj!=NULL);
assert(address_belongs_to_gc_heap(p_obj, gc));
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
}
root_set = pool_iterator_next(metadata->gc_rootset_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
marker->trace_stack = free_task_pool_get_entry(metadata);
/*second step: mark dirty pool*/
Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
while(dirty_set){
POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set);
while(!vector_block_iterator_end(dirty_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(dirty_set,iter);
assert(p_obj!=NULL); //FIXME: restrict condition?
obj_clear_dirty_in_table(p_obj);
obj_clear_mark_in_table(p_obj, marker);
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
num_dirtyset_slot ++;
}
vector_block_clear(dirty_set);
pool_put_entry(metadata->free_set_pool, dirty_set);
dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
/* third step: iterate over the mark tasks and scan objects */
marker->trace_stack = free_task_pool_get_entry(metadata);
Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool);
while(mark_task){
POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task);
while(!vector_block_iterator_end(mark_task,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter;
iter = vector_block_iterator_advance(mark_task,iter);
trace_object(marker, p_obj);
}
/* run out one task, put back to the pool and grab another task */
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
mark_task = pool_get_entry(metadata->mark_task_pool);
}
/* put back the last mark stack to the free pool */
mark_task = (Vector_Block*)marker->trace_stack;
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
marker->trace_stack = NULL;
//marker->time_mark += time_mark;
marker->num_dirty_slots_traced = num_dirtyset_slot;
//INFO2("gc.marker", "[final marker] processed dirty slot="<<num_dirtyset_slot);
return;
}
void wspace_mark_scan_mostly_concurrent(Conclctor* marker)
{
GC *gc = marker->gc;
GC_Metadata *metadata = gc->metadata;
unsigned int num_dirtyset_slot = 0;
marker->trace_stack = free_task_pool_get_entry(metadata);
/* first step: copy all root objects to mark tasks.*/
Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool);
while(root_set){
POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set);
while(!vector_block_iterator_end(root_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(root_set,iter);
assert(p_obj!=NULL);
assert(address_belongs_to_gc_heap(p_obj, gc));
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
}
root_set = pool_iterator_next(metadata->gc_rootset_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
marker->trace_stack = free_task_pool_get_entry(metadata);
/* following code has such concerns:
1, current_thread_id should be unique
2, mostly concurrent do not need adding new marker dynamically
3, when the heap is exhausted, final marking will enumeration rootset, it should be after above actions
*/
unsigned int current_thread_id = atomic_inc32(&num_active_markers);
if((current_thread_id+1) == gc->num_active_markers )
state_transformation( gc, GC_CON_START_MARKERS, GC_CON_TRACING);
while( gc->gc_concurrent_status == GC_CON_START_MARKERS );
retry:
/*second step: mark dirty pool*/
Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
while(dirty_set){
POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set);
while(!vector_block_iterator_end(dirty_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(dirty_set,iter);
assert(p_obj!=NULL); //FIXME: restrict condition?
obj_clear_dirty_in_table(p_obj);
obj_clear_mark_in_table(p_obj, marker);
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
num_dirtyset_slot ++;
}
vector_block_clear(dirty_set);
pool_put_entry(metadata->free_set_pool, dirty_set);
dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
/* third step: iterate over the mark tasks and scan objects */
marker->trace_stack = free_task_pool_get_entry(metadata);
Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool);
while(mark_task){
POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task);
while(!vector_block_iterator_end(mark_task,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter;
iter = vector_block_iterator_advance(mark_task,iter);
trace_object(marker, p_obj);
}
/* run out one task, put back to the pool and grab another task */
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
mark_task = pool_get_entry(metadata->mark_task_pool);
}
/*
if(current_thread_id == 0){
gc_prepare_dirty_set(marker->gc);
}*/
gc_copy_local_dirty_set_to_global(gc);
/* conditions to terminate mark:
1.All thread finished current job.
2.Flag is set to terminate concurrent mark.
*/
atomic_dec32(&num_active_markers);
while(num_active_markers != 0 || !concurrent_mark_need_terminating_mc(gc) ) {
if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)) {
atomic_inc32(&num_active_markers);
goto retry;
} else if( current_thread_id >= mostly_con_long_marker_num ) {
break;
}
apr_sleep(15000);
}
/*
while(num_active_markers != 0 || !concurrent_mark_need_terminating_mc(gc)){
if(!pool_is_empty(metadata->mark_task_pool) || !pool_is_empty(metadata->gc_dirty_set_pool)){
atomic_inc32(&num_active_markers);
goto retry;
}
}*/
/* put back the last mark stack to the free pool */
mark_task = (Vector_Block*)marker->trace_stack;
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
marker->trace_stack = NULL;
marker->num_dirty_slots_traced = num_dirtyset_slot;
/*
if(num_dirtyset_slot!=0) {
lock(info_lock);
INFO2("gc.marker", "marker ["<< current_thread_id <<"] processed dirty slot="<<num_dirtyset_slot);
unlock(info_lock);
}*/
return;
}
void wspace_mark_scan_concurrent(Conclctor* marker)
{
//marker->time_measurement_start = time_now();
GC *gc = marker->gc;
GC_Metadata *metadata = gc->metadata;
/* reset the num_finished_collectors to be 0 by one collector. This is necessary for the barrier later. */
unsigned int current_thread_id = atomic_inc32(&num_active_markers);
marker->trace_stack = free_task_pool_get_entry(metadata);
Vector_Block *root_set = pool_iterator_next(metadata->gc_rootset_pool);
/* first step: copy all root objects to mark tasks.*/
while(root_set){
POINTER_SIZE_INT *iter = vector_block_iterator_init(root_set);
while(!vector_block_iterator_end(root_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(root_set,iter);
assert(p_obj!=NULL);
assert(address_belongs_to_gc_heap(p_obj, gc));
//if(obj_mark_gray_in_table(p_obj, &root_set_obj_size))
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
}
root_set = pool_iterator_next(metadata->gc_rootset_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
marker->trace_stack = free_task_pool_get_entry(metadata);
state_transformation( gc, GC_CON_START_MARKERS, GC_CON_TRACING);
retry:
gc_copy_local_dirty_set_to_global(marker->gc);
/*second step: mark dirty object snapshot pool*/
Vector_Block* dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
while(dirty_set){
POINTER_SIZE_INT* iter = vector_block_iterator_init(dirty_set);
while(!vector_block_iterator_end(dirty_set,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object *)*iter;
iter = vector_block_iterator_advance(dirty_set,iter);
if(p_obj==NULL) { //FIXME: restrict?
RAISE_ERROR;
}
marker->num_dirty_slots_traced++;
if(obj_mark_gray_in_table(p_obj))
collector_tracestack_push((Collector*)marker, p_obj);
}
vector_block_clear(dirty_set);
pool_put_entry(metadata->free_set_pool, dirty_set);
dirty_set = pool_get_entry(metadata->gc_dirty_set_pool);
}
/* put back the last trace_stack task */
pool_put_entry(metadata->mark_task_pool, marker->trace_stack);
/* third step: iterate over the mark tasks and scan objects */
/* get a task buf for the mark stack */
marker->trace_stack = free_task_pool_get_entry(metadata);
Vector_Block *mark_task = pool_get_entry(metadata->mark_task_pool);
while(mark_task){
POINTER_SIZE_INT *iter = vector_block_iterator_init(mark_task);
while(!vector_block_iterator_end(mark_task,iter)){
Partial_Reveal_Object *p_obj = (Partial_Reveal_Object*)*iter;
iter = vector_block_iterator_advance(mark_task,iter);
trace_object(marker, p_obj);
}
/* run out one task, put back to the pool and grab another task */
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
mark_task = pool_get_entry(metadata->mark_task_pool);
}
/* termination condition:
1.all thread finished current job.
2.local snapshot vectors are empty.
3.global snapshot pool is empty.
*/
atomic_dec32(&num_active_markers);
while(num_active_markers != 0 || !concurrent_mark_need_terminating_otf(gc)){
if(!pool_is_empty(metadata->mark_task_pool) || !concurrent_mark_need_terminating_otf(gc)){
atomic_inc32(&num_active_markers);
goto retry;
}
apr_sleep(15000);
}
state_transformation( gc, GC_CON_TRACING, GC_CON_TRACE_DONE );
/* put back the last mark stack to the free pool */
mark_task = (Vector_Block*)marker->trace_stack;
vector_stack_clear(mark_task);
pool_put_entry(metadata->free_task_pool, mark_task);
marker->trace_stack = NULL;
assert(pool_is_empty(metadata->gc_dirty_set_pool));
//INFO2("gc.con.info", "<stage 5>first marker finishes its job");
return;
}
static FORCE_INLINE void forward_object(Collector *collector, REF *p_ref)
{
Space* space = collector->collect_space;
GC* gc = collector->gc;
Partial_Reveal_Object *p_obj = read_slot(p_ref);
/* p_obj can also be in tospace because this p_ref is a redundant one in mutator remset.
We don't rem p_ref because it was remembered in first time it's met.
FIXME:: the situation obj_belongs_to_tospace() should never be true if we
remember object rather than slot. Currently, mutator remembers objects, and
collector remembers slots. Although collectors remember slots, we are sure
there are no chances to have repetitive p_ref because an object is scanned only
when it is marked or forwarded atomically, hence only one collector has chance
to do the scanning. */
if(!obj_belongs_to_nos(p_obj) || obj_belongs_to_tospace(p_obj)) return;
Partial_Reveal_Object* p_target_obj = NULL;
Boolean to_rem_slot = FALSE;
/* Fastpath: object has already been forwarded, update the ref slot */
if(obj_is_fw_in_oi(p_obj)){
p_target_obj = obj_get_fw_in_oi(p_obj);
write_slot(p_ref, p_target_obj);
/* check if the target obj stays in NOS, and p_ref from MOS. If yes, rem p_ref. */
if(obj_belongs_to_tospace(p_target_obj))
if( !addr_belongs_to_nos(p_ref) && address_belongs_to_gc_heap(p_ref, gc))
collector_remset_add_entry(collector, ( Partial_Reveal_Object**) p_ref);
return;
}
/* following is the logic for forwarding */
p_target_obj = collector_forward_object(collector, p_obj);
/* if p_target_obj is NULL, it is forwarded by other thread.
Note: a race condition here, it might be forwarded by other, but not set the
forwarding pointer yet. We need spin here to get the forwarding pointer.
We can implement the collector_forward_object() so that the forwarding pointer
is set in the atomic instruction, which requires to roll back the mos_alloced
space. That is easy for thread local block allocation cancellation. */
if( p_target_obj == NULL ){
if(collector->result == FALSE ){
/* failed to forward, let's get back to controller. */
vector_stack_clear(collector->trace_stack);
return;
}
/* forwarded already*/
p_target_obj = obj_get_fw_in_oi(p_obj);
}else{ /* otherwise, we successfully forwarded */
#ifdef GC_GEN_STATS
if(gc_profile){
GC_Gen_Collector_Stats* stats = (GC_Gen_Collector_Stats*)collector->stats;
gc_gen_collector_update_marked_nos_obj_stats_minor(stats);
gc_gen_collector_update_moved_nos_obj_stats_minor(stats, vm_object_size(p_obj));
}
#endif
scan_object(collector, p_target_obj);
}
assert(p_target_obj);
write_slot(p_ref, p_target_obj);
/* check if the target obj stays in NOS, and p_ref from MOS. If yes, rem p_ref. */
if(obj_belongs_to_tospace(p_target_obj)){
if( !addr_belongs_to_nos(p_ref) && address_belongs_to_gc_heap(p_ref, gc))
collector_remset_add_entry(collector, ( Partial_Reveal_Object**) p_ref);
}
return;
}