void runtime_entersyscall(void) { uint32 v; // Leave SP around for gc and traceback. #ifdef USING_SPLIT_STACK g->gcstack = __splitstack_find(NULL, NULL, &g->gcstack_size, &g->gcnext_segment, &g->gcnext_sp, &g->gcinitial_sp); #else g->gcnext_sp = (byte *) &v; #endif // Save the registers in the g structure so that any pointers // held in registers will be seen by the garbage collector. // We could use getcontext here, but setjmp is more efficient // because it doesn't need to save the signal mask. setjmp(g->gcregs); g->status = Gsyscall; // Fast path. // The slow path inside the schedlock/schedunlock will get // through without stopping if it does: // mcpu-- // gwait not true // waitstop && mcpu <= mcpumax not true // If we can do the same with a single atomic add, // then we can skip the locks. v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift); if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v))) return; schedlock(); v = runtime_atomicload(&runtime_sched.atomic); if(atomic_gwaiting(v)) { matchmg(); v = runtime_atomicload(&runtime_sched.atomic); } if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) { runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift); runtime_notewakeup(&runtime_sched.stopped); } schedunlock(); }
// Mark g ready to run. Sched is already locked. // G might be running already and about to stop. // The sched lock protects g->status from changing underfoot. static void readylocked(G *g) { if(g->m){ // Running on another machine. // Ready it when it stops. g->readyonstop = 1; return; } // Mark runnable. if(g->status == Grunnable || g->status == Grunning) { runtime·printf("goroutine %d has status %d\n", g->goid, g->status); runtime·throw("bad g->status in ready"); } g->status = Grunnable; gput(g); matchmg(); }
// Implementation of runtime.GOMAXPROCS. // delete when scheduler is stronger int32 runtime·gomaxprocsfunc(int32 n) { int32 ret; uint32 v; schedlock(); ret = runtime·gomaxprocs; if(n <= 0) n = ret; if(n > maxgomaxprocs) n = maxgomaxprocs; runtime·gomaxprocs = n; if(runtime·gomaxprocs > 1) runtime·singleproc = false; if(runtime·gcwaiting != 0) { if(atomic_mcpumax(runtime·sched.atomic) != 1) runtime·throw("invalid mcpumax during gc"); schedunlock(); return ret; } setmcpumax(n); // If there are now fewer allowed procs // than procs running, stop. v = runtime·atomicload(&runtime·sched.atomic); if(atomic_mcpu(v) > n) { schedunlock(); runtime·gosched(); return ret; } // handle more procs matchmg(); schedunlock(); return ret; }
void runtime·starttheworld(void) { M *mp; int32 max; // Figure out how many CPUs GC could possibly use. max = runtime·gomaxprocs; if(max > runtime·ncpu) max = runtime·ncpu; if(max > MaxGcproc) max = MaxGcproc; schedlock(); runtime·gcwaiting = 0; setmcpumax(runtime·gomaxprocs); matchmg(); if(runtime·gcprocs() < max && canaddmcpu()) { // If GC could have used another helper proc, start one now, // in the hope that it will be available next time. // It would have been even better to start it before the collection, // but doing so requires allocating memory, so it's tricky to // coordinate. This lazy approach works out in practice: // we don't mind if the first couple gc rounds don't have quite // the maximum number of procs. // canaddmcpu above did mcpu++ // (necessary, because m will be doing various // initialization work so is definitely running), // but m is not running a specific goroutine, // so set the helpgc flag as a signal to m's // first schedule(nil) to mcpu-- and grunning--. mp = runtime·newm(); mp->helpgc = 1; runtime·sched.grunning++; } schedunlock(); }
// Get the next goroutine that m should run. // Sched must be locked on entry, is unlocked on exit. // Makes sure that at most $GOMAXPROCS g's are // running on cpus (not in system calls) at any given time. static G* nextgandunlock(void) { G *gp; uint32 v; top: if(atomic_mcpu(runtime·sched.atomic) >= maxgomaxprocs) runtime·throw("negative mcpu"); // If there is a g waiting as m->nextg, the mcpu++ // happened before it was passed to mnextg. if(m->nextg != nil) { gp = m->nextg; m->nextg = nil; schedunlock(); return gp; } if(m->lockedg != nil) { // We can only run one g, and it's not available. // Make sure some other cpu is running to handle // the ordinary run queue. if(runtime·sched.gwait != 0) { matchmg(); // m->lockedg might have been on the queue. if(m->nextg != nil) { gp = m->nextg; m->nextg = nil; schedunlock(); return gp; } } } else { // Look for work on global queue. while(haveg() && canaddmcpu()) { gp = gget(); if(gp == nil) runtime·throw("gget inconsistency"); if(gp->lockedm) { mnextg(gp->lockedm, gp); continue; } runtime·sched.grunning++; schedunlock(); return gp; } // The while loop ended either because the g queue is empty // or because we have maxed out our m procs running go // code (mcpu >= mcpumax). We need to check that // concurrent actions by entersyscall/exitsyscall cannot // invalidate the decision to end the loop. // // We hold the sched lock, so no one else is manipulating the // g queue or changing mcpumax. Entersyscall can decrement // mcpu, but if does so when there is something on the g queue, // the gwait bit will be set, so entersyscall will take the slow path // and use the sched lock. So it cannot invalidate our decision. // // Wait on global m queue. mput(m); } // Look for deadlock situation. // There is a race with the scavenger that causes false negatives: // if the scavenger is just starting, then we have // scvg != nil && grunning == 0 && gwait == 0 // and we do not detect a deadlock. It is possible that we should // add that case to the if statement here, but it is too close to Go 1 // to make such a subtle change. Instead, we work around the // false negative in trivial programs by calling runtime.gosched // from the main goroutine just before main.main. // See runtime·main above. // // On a related note, it is also possible that the scvg == nil case is // wrong and should include gwait, but that does not happen in // standard Go programs, which all start the scavenger. // if((scvg == nil && runtime·sched.grunning == 0) || (scvg != nil && runtime·sched.grunning == 1 && runtime·sched.gwait == 0 && (scvg->status == Grunning || scvg->status == Gsyscall))) { runtime·throw("all goroutines are asleep - deadlock!"); } m->nextg = nil; m->waitnextg = 1; runtime·noteclear(&m->havenextg); // Stoptheworld is waiting for all but its cpu to go to stop. // Entersyscall might have decremented mcpu too, but if so // it will see the waitstop and take the slow path. // Exitsyscall never increments mcpu beyond mcpumax. v = runtime·atomicload(&runtime·sched.atomic); if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) { // set waitstop = 0 (known to be 1) runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift); runtime·notewakeup(&runtime·sched.stopped); } schedunlock(); runtime·notesleep(&m->havenextg); if(m->helpgc) { runtime·gchelper(); m->helpgc = 0; runtime·lock(&runtime·sched); goto top; } if((gp = m->nextg) == nil) runtime·throw("bad m->nextg in nextgoroutine"); m->nextg = nil; return gp; }
// Get the next goroutine that m should run. // Sched must be locked on entry, is unlocked on exit. // Makes sure that at most $GOMAXPROCS g's are // running on cpus (not in system calls) at any given time. static G* nextgandunlock(void) { G *gp; uint32 v; top: if(atomic_mcpu(runtime_sched.atomic) >= maxgomaxprocs) runtime_throw("negative mcpu"); // If there is a g waiting as m->nextg, the mcpu++ // happened before it was passed to mnextg. if(m->nextg != nil) { gp = m->nextg; m->nextg = nil; schedunlock(); return gp; } if(m->lockedg != nil) { // We can only run one g, and it's not available. // Make sure some other cpu is running to handle // the ordinary run queue. if(runtime_sched.gwait != 0) { matchmg(); // m->lockedg might have been on the queue. if(m->nextg != nil) { gp = m->nextg; m->nextg = nil; schedunlock(); return gp; } } } else { // Look for work on global queue. while(haveg() && canaddmcpu()) { gp = gget(); if(gp == nil) runtime_throw("gget inconsistency"); if(gp->lockedm) { mnextg(gp->lockedm, gp); continue; } runtime_sched.grunning++; schedunlock(); return gp; } // The while loop ended either because the g queue is empty // or because we have maxed out our m procs running go // code (mcpu >= mcpumax). We need to check that // concurrent actions by entersyscall/exitsyscall cannot // invalidate the decision to end the loop. // // We hold the sched lock, so no one else is manipulating the // g queue or changing mcpumax. Entersyscall can decrement // mcpu, but if does so when there is something on the g queue, // the gwait bit will be set, so entersyscall will take the slow path // and use the sched lock. So it cannot invalidate our decision. // // Wait on global m queue. mput(m); } v = runtime_atomicload(&runtime_sched.atomic); if(runtime_sched.grunning == 0) runtime_throw("all goroutines are asleep - deadlock!"); m->nextg = nil; m->waitnextg = 1; runtime_noteclear(&m->havenextg); // Stoptheworld is waiting for all but its cpu to go to stop. // Entersyscall might have decremented mcpu too, but if so // it will see the waitstop and take the slow path. // Exitsyscall never increments mcpu beyond mcpumax. if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) { // set waitstop = 0 (known to be 1) runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift); runtime_notewakeup(&runtime_sched.stopped); } schedunlock(); runtime_notesleep(&m->havenextg); if(m->helpgc) { runtime_gchelper(); m->helpgc = 0; runtime_lock(&runtime_sched); goto top; } if((gp = m->nextg) == nil) runtime_throw("bad m->nextg in nextgoroutine"); m->nextg = nil; return gp; }