StgPtr
todo_block_full (uint32_t size, gen_workspace *ws)
{
bool urgent_to_push, can_extend;
StgPtr p;
bdescr *bd;
// todo_free has been pre-incremented by Evac.c:alloc_for_copy(). We
// are expected to leave it bumped when we've finished here.
ws->todo_free -= size;
bd = ws->todo_bd;
ASSERT(bd != NULL);
ASSERT(bd->link == NULL);
ASSERT(bd->gen == ws->gen);
// We intentionally set ws->todo_lim lower than the full size of
// the block, so that we can push out some work to the global list
// and get the parallel threads working as soon as possible.
//
// So when ws->todo_lim is reached, we end up here and have to
// decide whether it's worth pushing out the work we have or not.
// If we have enough room in the block to evacuate the current
// object, and it's not urgent to push this work, then we just
// extend the limit and keep going. Where "urgent" is defined as:
// the global pool is empty, and there's enough work in this block
// to make it worth pushing.
//
urgent_to_push =
looksEmptyWSDeque(ws->todo_q) &&
(ws->todo_free - bd->u.scan >= WORK_UNIT_WORDS / 2);
// We can extend the limit for the current block if there's enough
// room for the current object, *and* we're not into the second or
// subsequent block of a large block (see Note [big objects]).
can_extend =
ws->todo_free + size <= bd->start + bd->blocks * BLOCK_SIZE_W
&& ws->todo_free < ws->todo_bd->start + BLOCK_SIZE_W;
if (!urgent_to_push && can_extend)
{
ws->todo_lim = stg_min(bd->start + bd->blocks * BLOCK_SIZE_W,
ws->todo_lim + stg_max(WORK_UNIT_WORDS,size));
debugTrace(DEBUG_gc, "increasing limit for %p to %p",
bd->start, ws->todo_lim);
p = ws->todo_free;
ws->todo_free += size;
return p;
}
gct->copied += ws->todo_free - bd->free;
bd->free = ws->todo_free;
ASSERT(bd->u.scan >= bd->start && bd->u.scan <= bd->free);
// If this block is not the scan block, we want to push it out and
// make room for a new todo block.
if (bd != gct->scan_bd)
{
// If this block does not have enough space to allocate the
// current object, but it also doesn't have any work to push, then
// push it on to the scanned list.
if (bd->u.scan == bd->free)
{
if (bd->free == bd->start) {
// Normally the block would not be empty, because then
// there would be enough room to copy the current
// object. However, if the object we're copying is
// larger than a block, then we might have an empty
// block here.
freeGroup_sync(bd);
} else {
push_scanned_block(bd, ws);
}
}
// Otherwise, push this block out to the global list.
else
{
DEBUG_ONLY( generation *gen );
DEBUG_ONLY( gen = ws->gen );
debugTrace(DEBUG_gc, "push todo block %p (%ld words), step %d, todo_q: %ld",
bd->start, (unsigned long)(bd->free - bd->u.scan),
gen->no, dequeElements(ws->todo_q));
if (!pushWSDeque(ws->todo_q, bd)) {
bd->link = ws->todo_overflow;
ws->todo_overflow = bd;
ws->n_todo_overflow++;
}
}
}
ws->todo_bd = NULL;
ws->todo_free = NULL;
ws->todo_lim = NULL;
alloc_todo_block(ws, size);
p = ws->todo_free;
ws->todo_free += size;
return p;
}
static GNUC_ATTR_HOT void
scavenge_block (bdescr *bd)
{
StgPtr p, q;
StgInfoTable *info;
rtsBool saved_eager_promotion;
gen_workspace *ws;
debugTrace(DEBUG_gc, "scavenging block %p (gen %d) @ %p",
bd->start, bd->gen_no, bd->u.scan);
gct->scan_bd = bd;
gct->evac_gen_no = bd->gen_no;
saved_eager_promotion = gct->eager_promotion;
gct->failed_to_evac = rtsFalse;
ws = &gct->gens[bd->gen->no];
p = bd->u.scan;
// we might be evacuating into the very object that we're
// scavenging, so we have to check the real bd->free pointer each
// time around the loop.
while (p < bd->free || (bd == ws->todo_bd && p < ws->todo_free)) {
ASSERT(bd->link == NULL);
ASSERT(LOOKS_LIKE_CLOSURE_PTR(p));
info = get_itbl((StgClosure *)p);
ASSERT(gct->thunk_selector_depth == 0);
q = p;
switch (info->type) {
case MVAR_CLEAN:
case MVAR_DIRTY:
{
StgMVar *mvar = ((StgMVar *)p);
gct->eager_promotion = rtsFalse;
evacuate((StgClosure **)&mvar->head);
evacuate((StgClosure **)&mvar->tail);
evacuate((StgClosure **)&mvar->value);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) {
mvar->header.info = &stg_MVAR_DIRTY_info;
} else {
mvar->header.info = &stg_MVAR_CLEAN_info;
}
p += sizeofW(StgMVar);
break;
}
case FUN_2_0:
scavenge_fun_srt(info);
evacuate(&((StgClosure *)p)->payload[1]);
evacuate(&((StgClosure *)p)->payload[0]);
p += sizeofW(StgHeader) + 2;
break;
case THUNK_2_0:
scavenge_thunk_srt(info);
evacuate(&((StgThunk *)p)->payload[1]);
evacuate(&((StgThunk *)p)->payload[0]);
p += sizeofW(StgThunk) + 2;
break;
case CONSTR_2_0:
evacuate(&((StgClosure *)p)->payload[1]);
evacuate(&((StgClosure *)p)->payload[0]);
p += sizeofW(StgHeader) + 2;
break;
case THUNK_1_0:
scavenge_thunk_srt(info);
evacuate(&((StgThunk *)p)->payload[0]);
p += sizeofW(StgThunk) + 1;
break;
case FUN_1_0:
scavenge_fun_srt(info);
case CONSTR_1_0:
evacuate(&((StgClosure *)p)->payload[0]);
p += sizeofW(StgHeader) + 1;
break;
case THUNK_0_1:
scavenge_thunk_srt(info);
p += sizeofW(StgThunk) + 1;
break;
case FUN_0_1:
scavenge_fun_srt(info);
case CONSTR_0_1:
p += sizeofW(StgHeader) + 1;
break;
case THUNK_0_2:
scavenge_thunk_srt(info);
p += sizeofW(StgThunk) + 2;
break;
case FUN_0_2:
scavenge_fun_srt(info);
case CONSTR_0_2:
p += sizeofW(StgHeader) + 2;
break;
case THUNK_1_1:
scavenge_thunk_srt(info);
evacuate(&((StgThunk *)p)->payload[0]);
p += sizeofW(StgThunk) + 2;
break;
case FUN_1_1:
scavenge_fun_srt(info);
case CONSTR_1_1:
evacuate(&((StgClosure *)p)->payload[0]);
p += sizeofW(StgHeader) + 2;
break;
case FUN:
scavenge_fun_srt(info);
goto gen_obj;
case THUNK:
{
StgPtr end;
scavenge_thunk_srt(info);
end = (P_)((StgThunk *)p)->payload + info->layout.payload.ptrs;
for (p = (P_)((StgThunk *)p)->payload; p < end; p++) {
evacuate((StgClosure **)p);
}
p += info->layout.payload.nptrs;
break;
}
gen_obj:
case CONSTR:
case WEAK:
case PRIM:
{
StgPtr end;
end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
evacuate((StgClosure **)p);
}
p += info->layout.payload.nptrs;
break;
}
case BCO: {
StgBCO *bco = (StgBCO *)p;
evacuate((StgClosure **)&bco->instrs);
evacuate((StgClosure **)&bco->literals);
evacuate((StgClosure **)&bco->ptrs);
p += bco_sizeW(bco);
break;
}
case IND_PERM:
case BLACKHOLE:
evacuate(&((StgInd *)p)->indirectee);
p += sizeofW(StgInd);
break;
case MUT_VAR_CLEAN:
case MUT_VAR_DIRTY:
gct->eager_promotion = rtsFalse;
evacuate(&((StgMutVar *)p)->var);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) {
((StgClosure *)q)->header.info = &stg_MUT_VAR_DIRTY_info;
} else {
((StgClosure *)q)->header.info = &stg_MUT_VAR_CLEAN_info;
}
p += sizeofW(StgMutVar);
break;
case BLOCKING_QUEUE:
{
StgBlockingQueue *bq = (StgBlockingQueue *)p;
gct->eager_promotion = rtsFalse;
evacuate(&bq->bh);
evacuate((StgClosure**)&bq->owner);
evacuate((StgClosure**)&bq->queue);
evacuate((StgClosure**)&bq->link);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) {
bq->header.info = &stg_BLOCKING_QUEUE_DIRTY_info;
} else {
bq->header.info = &stg_BLOCKING_QUEUE_CLEAN_info;
}
p += sizeofW(StgBlockingQueue);
break;
}
case THUNK_SELECTOR:
{
StgSelector *s = (StgSelector *)p;
evacuate(&s->selectee);
p += THUNK_SELECTOR_sizeW();
break;
}
// A chunk of stack saved in a heap object
case AP_STACK:
{
StgAP_STACK *ap = (StgAP_STACK *)p;
evacuate(&ap->fun);
scavenge_stack((StgPtr)ap->payload, (StgPtr)ap->payload + ap->size);
p = (StgPtr)ap->payload + ap->size;
break;
}
case PAP:
p = scavenge_PAP((StgPAP *)p);
break;
case AP:
p = scavenge_AP((StgAP *)p);
break;
case ARR_WORDS:
// nothing to follow
p += arr_words_sizeW((StgArrWords *)p);
break;
case MUT_ARR_PTRS_CLEAN:
case MUT_ARR_PTRS_DIRTY:
{
// We don't eagerly promote objects pointed to by a mutable
// array, but if we find the array only points to objects in
// the same or an older generation, we mark it "clean" and
// avoid traversing it during minor GCs.
gct->eager_promotion = rtsFalse;
p = scavenge_mut_arr_ptrs((StgMutArrPtrs*)p);
if (gct->failed_to_evac) {
((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_DIRTY_info;
} else {
((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_CLEAN_info;
}
gct->eager_promotion = saved_eager_promotion;
gct->failed_to_evac = rtsTrue; // always put it on the mutable list.
break;
}
case MUT_ARR_PTRS_FROZEN:
case MUT_ARR_PTRS_FROZEN0:
// follow everything
{
p = scavenge_mut_arr_ptrs((StgMutArrPtrs*)p);
// If we're going to put this object on the mutable list, then
// set its info ptr to MUT_ARR_PTRS_FROZEN0 to indicate that.
if (gct->failed_to_evac) {
((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_FROZEN0_info;
} else {
((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_FROZEN_info;
}
break;
}
case TSO:
{
scavengeTSO((StgTSO *)p);
p += sizeofW(StgTSO);
break;
}
case STACK:
{
StgStack *stack = (StgStack*)p;
gct->eager_promotion = rtsFalse;
scavenge_stack(stack->sp, stack->stack + stack->stack_size);
stack->dirty = gct->failed_to_evac;
p += stack_sizeW(stack);
gct->eager_promotion = saved_eager_promotion;
break;
}
case MUT_PRIM:
{
StgPtr end;
gct->eager_promotion = rtsFalse;
end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
evacuate((StgClosure **)p);
}
p += info->layout.payload.nptrs;
gct->eager_promotion = saved_eager_promotion;
gct->failed_to_evac = rtsTrue; // mutable
break;
}
case TREC_CHUNK:
{
StgWord i;
StgTRecChunk *tc = ((StgTRecChunk *) p);
TRecEntry *e = &(tc -> entries[0]);
gct->eager_promotion = rtsFalse;
evacuate((StgClosure **)&tc->prev_chunk);
for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
evacuate((StgClosure **)&e->tvar);
evacuate((StgClosure **)&e->expected_value);
evacuate((StgClosure **)&e->new_value);
}
gct->eager_promotion = saved_eager_promotion;
gct->failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgTRecChunk);
break;
}
default:
barf("scavenge: unimplemented/strange closure type %d @ %p",
info->type, p);
}
/*
* We need to record the current object on the mutable list if
* (a) It is actually mutable, or
* (b) It contains pointers to a younger generation.
* Case (b) arises if we didn't manage to promote everything that
* the current object points to into the current generation.
*/
if (gct->failed_to_evac) {
gct->failed_to_evac = rtsFalse;
if (bd->gen_no > 0) {
recordMutableGen_GC((StgClosure *)q, bd->gen_no);
}
}
}
if (p > bd->free) {
gct->copied += ws->todo_free - bd->free;
bd->free = p;
}
debugTrace(DEBUG_gc, " scavenged %ld bytes",
(unsigned long)((bd->free - bd->u.scan) * sizeof(W_)));
// update stats: this is a block that has been scavenged
gct->scanned += bd->free - bd->u.scan;
bd->u.scan = bd->free;
if (bd != ws->todo_bd) {
// we're not going to evac any more objects into
// this block, so push it now.
push_scanned_block(bd, ws);
}
gct->scan_bd = NULL;
}
