void sendMessage(Capability *from_cap, Capability *to_cap, Message *msg) { ACQUIRE_LOCK(&to_cap->lock); #ifdef DEBUG { const StgInfoTable *i = msg->header.info; if (i != &stg_MSG_THROWTO_info && i != &stg_MSG_BLACKHOLE_info && i != &stg_MSG_TRY_WAKEUP_info && i != &stg_IND_info && // can happen if a MSG_BLACKHOLE is revoked i != &stg_WHITEHOLE_info) { barf("sendMessage: %p", i); } } #endif msg->link = to_cap->inbox; to_cap->inbox = msg; recordClosureMutated(from_cap,(StgClosure*)msg); if (to_cap->running_task == NULL) { to_cap->running_task = myTask(); // precond for releaseCapability_() releaseCapability_(to_cap,rtsFalse); } else { interruptCapability(to_cap); } RELEASE_LOCK(&to_cap->lock); }
void dirty_STACK (Capability *cap, StgStack *stack) { if (stack->dirty == 0) { stack->dirty = 1; recordClosureMutated(cap,(StgClosure*)stack); } }
void dirty_TSO (Capability *cap, StgTSO *tso) { if (tso->dirty == 0) { tso->dirty = 1; recordClosureMutated(cap,(StgClosure*)tso); } }
void dirty_TVAR(Capability *cap, StgTVar *p) { if (p->header.info == &stg_TVAR_CLEAN_info) { p->header.info = &stg_TVAR_DIRTY_info; recordClosureMutated(cap,(StgClosure*)p); } }
void setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target) { if (tso->dirty == 0) { tso->dirty = 1; recordClosureMutated(cap,(StgClosure*)tso); } tso->block_info.prev = target; }
// Setting a TSO's link field with a write barrier. // It is *not* necessary to call this function when // * setting the link field to END_TSO_QUEUE // * setting the link field of the currently running TSO, as it // will already be dirty. void setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target) { if (tso->dirty == 0) { tso->dirty = 1; recordClosureMutated(cap,(StgClosure*)tso); } tso->_link = target; }
/* This is the write barrier for MUT_VARs, a.k.a. IORefs. A MUT_VAR_CLEAN object is not on the mutable list; a MUT_VAR_DIRTY is. When written to, a MUT_VAR_CLEAN turns into a MUT_VAR_DIRTY and is put on the mutable list. */ void dirty_MUT_VAR(StgRegTable *reg, StgClosure *p) { Capability *cap = regTableToCapability(reg); if (p->header.info == &stg_MUT_VAR_CLEAN_info) { p->header.info = &stg_MUT_VAR_DIRTY_info; recordClosureMutated(cap,p); } }
void insertCompactHash (Capability *cap, StgCompactNFData *str, StgClosure *p, StgClosure *to) { insertHashTable(str->hash, (StgWord)p, (const void*)to); if (str->header.info == &stg_COMPACT_NFDATA_CLEAN_info) { str->header.info = &stg_COMPACT_NFDATA_DIRTY_info; recordClosureMutated(cap, (StgClosure*)str); } }
/* This is the write barrier for MVARs. An MVAR_CLEAN objects is not on the mutable list; a MVAR_DIRTY is. When written to, a MVAR_CLEAN turns into a MVAR_DIRTY and is put on the mutable list. The check for MVAR_CLEAN is inlined at the call site for speed, this really does make a difference on concurrency-heavy benchmarks such as Chaneneos and cheap-concurrency. */ void dirty_MVAR(StgRegTable *reg, StgClosure *p) { recordClosureMutated(regTableToCapability(reg),p); }
/* ----------------------------------------------------------------------------- * Pausing a thread * * We have to prepare for GC - this means doing lazy black holing * here. We also take the opportunity to do stack squeezing if it's * turned on. * -------------------------------------------------------------------------- */ void threadPaused(Capability *cap, StgTSO *tso) { StgClosure *frame; const StgRetInfoTable *info; const StgInfoTable *bh_info; const StgInfoTable *cur_bh_info USED_IF_THREADS; StgClosure *bh; StgPtr stack_end; uint32_t words_to_squeeze = 0; uint32_t weight = 0; uint32_t weight_pending = 0; bool prev_was_update_frame = false; StgWord heuristic_says_squeeze; // Check to see whether we have threads waiting to raise // exceptions, and we're not blocking exceptions, or are blocked // interruptibly. This is important; if a thread is running with // TSO_BLOCKEX and becomes blocked interruptibly, this is the only // place we ensure that the blocked_exceptions get a chance. maybePerformBlockedException (cap, tso); if (tso->what_next == ThreadKilled) { return; } // NB. Updatable thunks *must* be blackholed, either by eager blackholing or // lazy blackholing. See Note [upd-black-hole] in sm/Scav.c. stack_end = tso->stackobj->stack + tso->stackobj->stack_size; frame = (StgClosure *)tso->stackobj->sp; while ((P_)frame < stack_end) { info = get_ret_itbl(frame); switch (info->i.type) { case UPDATE_FRAME: // If we've already marked this frame, then stop here. if (frame->header.info == (StgInfoTable *)&stg_marked_upd_frame_info) { if (prev_was_update_frame) { words_to_squeeze += sizeofW(StgUpdateFrame); weight += weight_pending; weight_pending = 0; } goto end; } SET_INFO(frame, (StgInfoTable *)&stg_marked_upd_frame_info); bh = ((StgUpdateFrame *)frame)->updatee; bh_info = bh->header.info; #if defined(THREADED_RTS) retry: #endif // Note [suspend duplicate work] // // If the info table is a WHITEHOLE or a BLACKHOLE, then // another thread has claimed it (via the SET_INFO() // below), or is in the process of doing so. In that case // we want to suspend the work that the current thread has // done on this thunk and wait until the other thread has // finished. // // If eager blackholing is taking place, it could be the // case that the blackhole points to the current // TSO. e.g.: // // this thread other thread // -------------------------------------------------------- // c->indirectee = other_tso; // c->header.info = EAGER_BH // threadPaused(): // c->header.info = WHITEHOLE // c->indirectee = other_tso // c->indirectee = this_tso; // c->header.info = EAGER_BH // c->header.info = BLACKHOLE // threadPaused() // *** c->header.info is now BLACKHOLE, // c->indirectee points to this_tso // // So in this case do *not* suspend the work of the // current thread, because the current thread will become // deadlocked on itself. See #5226 for an instance of // this bug. // // Note that great care is required when entering computations // suspended by this mechanism. See Note [AP_STACKs must be eagerly // blackholed] for details. if (((bh_info == &stg_BLACKHOLE_info) && ((StgInd*)bh)->indirectee != (StgClosure*)tso) || (bh_info == &stg_WHITEHOLE_info)) { debugTrace(DEBUG_squeeze, "suspending duplicate work: %ld words of stack", (long)((StgPtr)frame - tso->stackobj->sp)); // If this closure is already an indirection, then // suspend the computation up to this point. // NB. check raiseAsync() to see what happens when // we're in a loop (#2783). suspendComputation(cap,tso,(StgUpdateFrame*)frame); // Now drop the update frame, and arrange to return // the value to the frame underneath: tso->stackobj->sp = (StgPtr)frame + sizeofW(StgUpdateFrame) - 2; tso->stackobj->sp[1] = (StgWord)bh; ASSERT(bh->header.info != &stg_TSO_info); tso->stackobj->sp[0] = (W_)&stg_enter_info; // And continue with threadPaused; there might be // yet more computation to suspend. frame = (StgClosure *)(tso->stackobj->sp + 2); prev_was_update_frame = false; continue; } // We should never have made it here in the event of blackholes that // we already own; they should have been marked when we blackholed // them and consequently we should have stopped our stack walk // above. ASSERT(!((bh_info == &stg_BLACKHOLE_info) && (((StgInd*)bh)->indirectee == (StgClosure*)tso))); // zero out the slop so that the sanity checker can tell // where the next closure is. OVERWRITING_CLOSURE(bh); // an EAGER_BLACKHOLE or CAF_BLACKHOLE gets turned into a // BLACKHOLE here. #if defined(THREADED_RTS) // first we turn it into a WHITEHOLE to claim it, and if // successful we write our TSO and then the BLACKHOLE info pointer. cur_bh_info = (const StgInfoTable *) cas((StgVolatilePtr)&bh->header.info, (StgWord)bh_info, (StgWord)&stg_WHITEHOLE_info); if (cur_bh_info != bh_info) { bh_info = cur_bh_info; goto retry; } #endif // The payload of the BLACKHOLE points to the TSO ((StgInd *)bh)->indirectee = (StgClosure *)tso; write_barrier(); SET_INFO(bh,&stg_BLACKHOLE_info); // .. and we need a write barrier, since we just mutated the closure: recordClosureMutated(cap,bh); // We pretend that bh has just been created. LDV_RECORD_CREATE(bh); frame = (StgClosure *) ((StgUpdateFrame *)frame + 1); if (prev_was_update_frame) { words_to_squeeze += sizeofW(StgUpdateFrame); weight += weight_pending; weight_pending = 0; } prev_was_update_frame = true; break; case UNDERFLOW_FRAME: case STOP_FRAME: goto end; // normal stack frames; do nothing except advance the pointer default: { uint32_t frame_size = stack_frame_sizeW(frame); weight_pending += frame_size; frame = (StgClosure *)((StgPtr)frame + frame_size); prev_was_update_frame = false; } } } end: // Should we squeeze or not? Arbitrary heuristic: we squeeze if // the number of words we have to shift down is less than the // number of stack words we squeeze away by doing so. // The threshold was bumped from 5 to 8 as a result of #2797 heuristic_says_squeeze = ((weight <= 8 && words_to_squeeze > 0) || weight < words_to_squeeze); debugTrace(DEBUG_squeeze, "words_to_squeeze: %d, weight: %d, squeeze: %s", words_to_squeeze, weight, heuristic_says_squeeze ? "YES" : "NO"); if (RtsFlags.GcFlags.squeezeUpdFrames == true && heuristic_says_squeeze) { stackSqueeze(cap, tso, (StgPtr)frame); tso->flags |= TSO_SQUEEZED; // This flag tells threadStackOverflow() that the stack was // squeezed, because it may not need to be expanded. } else { tso->flags &= ~TSO_SQUEEZED; } }
nat messageBlackHole(Capability *cap, MessageBlackHole *msg) { const StgInfoTable *info; StgClosure *p; StgBlockingQueue *bq; StgClosure *bh = UNTAG_CLOSURE(msg->bh); StgTSO *owner; debugTraceCap(DEBUG_sched, cap, "message: thread %d blocking on blackhole %p", (lnat)msg->tso->id, msg->bh); info = bh->header.info; // If we got this message in our inbox, it might be that the // BLACKHOLE has already been updated, and GC has shorted out the // indirection, so the pointer no longer points to a BLACKHOLE at // all. if (info != &stg_BLACKHOLE_info && info != &stg_CAF_BLACKHOLE_info && info != &__stg_EAGER_BLACKHOLE_info && info != &stg_WHITEHOLE_info) { // if it is a WHITEHOLE, then a thread is in the process of // trying to BLACKHOLE it. But we know that it was once a // BLACKHOLE, so there is at least a valid pointer in the // payload, so we can carry on. return 0; } // The blackhole must indirect to a TSO, a BLOCKING_QUEUE, an IND, // or a value. loop: // NB. VOLATILE_LOAD(), because otherwise gcc hoists the load // and turns this into an infinite loop. p = UNTAG_CLOSURE((StgClosure*)VOLATILE_LOAD(&((StgInd*)bh)->indirectee)); info = p->header.info; if (info == &stg_IND_info) { // This could happen, if e.g. we got a BLOCKING_QUEUE that has // just been replaced with an IND by another thread in // updateThunk(). In which case, if we read the indirectee // again we should get the value. goto loop; } else if (info == &stg_TSO_info) { owner = (StgTSO*)p; #ifdef THREADED_RTS if (owner->cap != cap) { sendMessage(cap, owner->cap, (Message*)msg); debugTraceCap(DEBUG_sched, cap, "forwarding message to cap %d", owner->cap->no); return 1; } #endif // owner is the owner of the BLACKHOLE, and resides on this // Capability. msg->tso is the first thread to block on this // BLACKHOLE, so we first create a BLOCKING_QUEUE object. bq = (StgBlockingQueue*)allocate(cap, sizeofW(StgBlockingQueue)); // initialise the BLOCKING_QUEUE object SET_HDR(bq, &stg_BLOCKING_QUEUE_DIRTY_info, CCS_SYSTEM); bq->bh = bh; bq->queue = msg; bq->owner = owner; msg->link = (MessageBlackHole*)END_TSO_QUEUE; // All BLOCKING_QUEUES are linked in a list on owner->bq, so // that we can search through them in the event that there is // a collision to update a BLACKHOLE and a BLOCKING_QUEUE // becomes orphaned (see updateThunk()). bq->link = owner->bq; owner->bq = bq; dirty_TSO(cap, owner); // we modified owner->bq // If the owner of the blackhole is currently runnable, then // bump it to the front of the run queue. This gives the // blocked-on thread a little boost which should help unblock // this thread, and may avoid a pile-up of other threads // becoming blocked on the same BLACKHOLE (#3838). // // NB. we check to make sure that the owner is not the same as // the current thread, since in that case it will not be on // the run queue. if (owner->why_blocked == NotBlocked && owner->id != msg->tso->id) { removeFromRunQueue(cap, owner); pushOnRunQueue(cap,owner); } // point to the BLOCKING_QUEUE from the BLACKHOLE write_barrier(); // make the BQ visible ((StgInd*)bh)->indirectee = (StgClosure *)bq; recordClosureMutated(cap,bh); // bh was mutated debugTraceCap(DEBUG_sched, cap, "thread %d blocked on thread %d", (lnat)msg->tso->id, (lnat)owner->id); return 1; // blocked } else if (info == &stg_BLOCKING_QUEUE_CLEAN_info || info == &stg_BLOCKING_QUEUE_DIRTY_info) { StgBlockingQueue *bq = (StgBlockingQueue *)p; ASSERT(bq->bh == bh); owner = bq->owner; ASSERT(owner != END_TSO_QUEUE); #ifdef THREADED_RTS if (owner->cap != cap) { sendMessage(cap, owner->cap, (Message*)msg); debugTraceCap(DEBUG_sched, cap, "forwarding message to cap %d", owner->cap->no); return 1; } #endif msg->link = bq->queue; bq->queue = msg; recordClosureMutated(cap,(StgClosure*)msg); if (info == &stg_BLOCKING_QUEUE_CLEAN_info) { bq->header.info = &stg_BLOCKING_QUEUE_DIRTY_info; recordClosureMutated(cap,(StgClosure*)bq); } debugTraceCap(DEBUG_sched, cap, "thread %d blocked on thread %d", (lnat)msg->tso->id, (lnat)owner->id); // See above, #3838 if (owner->why_blocked == NotBlocked && owner->id != msg->tso->id) { removeFromRunQueue(cap, owner); pushOnRunQueue(cap,owner); } return 1; // blocked } return 0; // not blocked }