void
runtime·starttheworld(bool extra)
{
M *m;
schedlock();
runtime·gcwaiting = 0;
setmcpumax(runtime·gomaxprocs);
matchmg();
if(extra && canaddmcpu()) {
// Start a new m that will (we hope) be idle
// and so available to help when the next
// garbage collection happens.
// canaddmcpu above did mcpu++
// (necessary, because m will be doing various
// initialization work so is definitely running),
// but m is not running a specific goroutine,
// so set the helpgc flag as a signal to m's
// first schedule(nil) to mcpu-- and grunning--.
m = runtime·newm();
m->helpgc = 1;
runtime·sched.grunning++;
}
schedunlock();
}
// Put on `g' queue. Sched must be locked.
static void
gput(G *g)
{
M *m;
// If g is wired, hand it off directly.
if((m = g->lockedm) != nil && canaddmcpu()) {
mnextg(m, g);
return;
}
// If g is the idle goroutine for an m, hand it off.
if(g->idlem != nil) {
if(g->idlem->idleg != nil) {
runtime·printf("m%d idle out of sync: g%d g%d\n",
g->idlem->id,
g->idlem->idleg->goid, g->goid);
runtime·throw("runtime: double idle");
}
g->idlem->idleg = g;
return;
}
g->schedlink = nil;
if(runtime·sched.ghead == nil)
runtime·sched.ghead = g;
else
runtime·sched.gtail->schedlink = g;
runtime·sched.gtail = g;
// increment gwait.
// if it transitions to nonzero, set atomic gwaiting bit.
if(runtime·sched.gwait++ == 0)
runtime·xadd(&runtime·sched.atomic, 1<<gwaitingShift);
}
// The bootstrap sequence is:
//
// call osinit
// call schedinit
// make & queue new G
// call runtime_mstart
//
// The new G calls runtime_main.
void
runtime_schedinit(void)
{
int32 n;
const byte *p;
m = &runtime_m0;
g = &runtime_g0;
m->g0 = g;
m->curg = g;
g->m = m;
initcontext();
inittlssize();
m->nomemprof++;
runtime_mallocinit();
mcommoninit(m);
runtime_goargs();
runtime_goenvs();
// For debugging:
// Allocate internal symbol table representation now,
// so that we don't need to call malloc when we crash.
// runtime_findfunc(0);
runtime_gomaxprocs = 1;
p = runtime_getenv("GOMAXPROCS");
if(p != nil && (n = runtime_atoi(p)) != 0) {
if(n > maxgomaxprocs)
n = maxgomaxprocs;
runtime_gomaxprocs = n;
}
// wait for the main goroutine to start before taking
// GOMAXPROCS into account.
setmcpumax(1);
runtime_singleproc = runtime_gomaxprocs == 1;
canaddmcpu(); // mcpu++ to account for bootstrap m
m->helpgc = 1; // flag to tell schedule() to mcpu--
runtime_sched.grunning++;
// Can not enable GC until all roots are registered.
// mstats.enablegc = 1;
m->nomemprof--;
}
// Kick off new m's as needed (up to mcpumax).
// Sched is locked.
static void
matchmg(void)
{
G *gp;
M *mp;
if(m->mallocing || m->gcing)
return;
while(haveg() && canaddmcpu()) {
gp = gget();
if(gp == nil)
runtime·throw("gget inconsistency");
// Find the m that will run gp.
if((mp = mget(gp)) == nil)
mp = runtime·newm();
mnextg(mp, gp);
}
}
// The bootstrap sequence is:
//
// call osinit
// call schedinit
// make & queue new G
// call runtime·mstart
//
// The new G calls runtime·main.
void
runtime·schedinit(void)
{
int32 n;
byte *p;
m->nomemprof++;
runtime·mprofinit();
runtime·mallocinit();
mcommoninit(m);
runtime·goargs();
runtime·goenvs();
// For debugging:
// Allocate internal symbol table representation now,
// so that we don't need to call malloc when we crash.
// runtime·findfunc(0);
runtime·gomaxprocs = 1;
p = runtime·getenv("GOMAXPROCS");
if(p != nil && (n = runtime·atoi(p)) != 0) {
if(n > maxgomaxprocs)
n = maxgomaxprocs;
runtime·gomaxprocs = n;
}
// wait for the main goroutine to start before taking
// GOMAXPROCS into account.
setmcpumax(1);
runtime·singleproc = runtime·gomaxprocs == 1;
canaddmcpu(); // mcpu++ to account for bootstrap m
m->helpgc = 1; // flag to tell schedule() to mcpu--
runtime·sched.grunning++;
mstats.enablegc = 1;
m->nomemprof--;
if(raceenabled)
runtime·raceinit();
}
void
runtime·starttheworld(void)
{
M *mp;
int32 max;
// Figure out how many CPUs GC could possibly use.
max = runtime·gomaxprocs;
if(max > runtime·ncpu)
max = runtime·ncpu;
if(max > MaxGcproc)
max = MaxGcproc;
schedlock();
runtime·gcwaiting = 0;
setmcpumax(runtime·gomaxprocs);
matchmg();
if(runtime·gcprocs() < max && canaddmcpu()) {
// If GC could have used another helper proc, start one now,
// in the hope that it will be available next time.
// It would have been even better to start it before the collection,
// but doing so requires allocating memory, so it's tricky to
// coordinate. This lazy approach works out in practice:
// we don't mind if the first couple gc rounds don't have quite
// the maximum number of procs.
// canaddmcpu above did mcpu++
// (necessary, because m will be doing various
// initialization work so is definitely running),
// but m is not running a specific goroutine,
// so set the helpgc flag as a signal to m's
// first schedule(nil) to mcpu-- and grunning--.
mp = runtime·newm();
mp->helpgc = 1;
runtime·sched.grunning++;
}
schedunlock();
}
// The bootstrap sequence is:
//
// call osinit
// call schedinit
// make & queue new G
// call runtime·mstart
//
// The new G calls runtime·main.
void
runtime·schedinit(void)
{
int32 n;
byte *p;
m->nomemprof++;
runtime·mallocinit();
mcommoninit(m);
runtime·goargs();
runtime·goenvs();
// For debugging:
// Allocate internal symbol table representation now,
// so that we don't need to call malloc when we crash.
// runtime·findfunc(0);
runtime·gomaxprocs = 1;
p = runtime·getenv("GOMAXPROCS");
if(p != nil && (n = runtime·atoi(p)) != 0) {
if(n > maxgomaxprocs)
n = maxgomaxprocs;
runtime·gomaxprocs = n;
}
setmcpumax(runtime·gomaxprocs);
runtime·singleproc = runtime·gomaxprocs == 1;
canaddmcpu(); // mcpu++ to account for bootstrap m
m->helpgc = 1; // flag to tell schedule() to mcpu--
runtime·sched.grunning++;
mstats.enablegc = 1;
m->nomemprof--;
scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runtime·schedinit);
}
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
// Makes sure that at most $GOMAXPROCS g's are
// running on cpus (not in system calls) at any given time.
static G*
nextgandunlock(void)
{
G *gp;
uint32 v;
top:
if(atomic_mcpu(runtime·sched.atomic) >= maxgomaxprocs)
runtime·throw("negative mcpu");
// If there is a g waiting as m->nextg, the mcpu++
// happened before it was passed to mnextg.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
if(m->lockedg != nil) {
// We can only run one g, and it's not available.
// Make sure some other cpu is running to handle
// the ordinary run queue.
if(runtime·sched.gwait != 0) {
matchmg();
// m->lockedg might have been on the queue.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
}
} else {
// Look for work on global queue.
while(haveg() && canaddmcpu()) {
gp = gget();
if(gp == nil)
runtime·throw("gget inconsistency");
if(gp->lockedm) {
mnextg(gp->lockedm, gp);
continue;
}
runtime·sched.grunning++;
schedunlock();
return gp;
}
// The while loop ended either because the g queue is empty
// or because we have maxed out our m procs running go
// code (mcpu >= mcpumax). We need to check that
// concurrent actions by entersyscall/exitsyscall cannot
// invalidate the decision to end the loop.
//
// We hold the sched lock, so no one else is manipulating the
// g queue or changing mcpumax. Entersyscall can decrement
// mcpu, but if does so when there is something on the g queue,
// the gwait bit will be set, so entersyscall will take the slow path
// and use the sched lock. So it cannot invalidate our decision.
//
// Wait on global m queue.
mput(m);
}
// Look for deadlock situation.
// There is a race with the scavenger that causes false negatives:
// if the scavenger is just starting, then we have
// scvg != nil && grunning == 0 && gwait == 0
// and we do not detect a deadlock. It is possible that we should
// add that case to the if statement here, but it is too close to Go 1
// to make such a subtle change. Instead, we work around the
// false negative in trivial programs by calling runtime.gosched
// from the main goroutine just before main.main.
// See runtime·main above.
//
// On a related note, it is also possible that the scvg == nil case is
// wrong and should include gwait, but that does not happen in
// standard Go programs, which all start the scavenger.
//
if((scvg == nil && runtime·sched.grunning == 0) ||
(scvg != nil && runtime·sched.grunning == 1 && runtime·sched.gwait == 0 &&
(scvg->status == Grunning || scvg->status == Gsyscall))) {
runtime·throw("all goroutines are asleep - deadlock!");
}
m->nextg = nil;
m->waitnextg = 1;
runtime·noteclear(&m->havenextg);
// Stoptheworld is waiting for all but its cpu to go to stop.
// Entersyscall might have decremented mcpu too, but if so
// it will see the waitstop and take the slow path.
// Exitsyscall never increments mcpu beyond mcpumax.
v = runtime·atomicload(&runtime·sched.atomic);
if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
// set waitstop = 0 (known to be 1)
runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
runtime·notewakeup(&runtime·sched.stopped);
}
schedunlock();
runtime·notesleep(&m->havenextg);
if(m->helpgc) {
runtime·gchelper();
m->helpgc = 0;
runtime·lock(&runtime·sched);
goto top;
}
if((gp = m->nextg) == nil)
runtime·throw("bad m->nextg in nextgoroutine");
m->nextg = nil;
return gp;
}
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
// Makes sure that at most $GOMAXPROCS g's are
// running on cpus (not in system calls) at any given time.
static G*
nextgandunlock(void)
{
G *gp;
uint32 v;
top:
if(atomic_mcpu(runtime_sched.atomic) >= maxgomaxprocs)
runtime_throw("negative mcpu");
// If there is a g waiting as m->nextg, the mcpu++
// happened before it was passed to mnextg.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
if(m->lockedg != nil) {
// We can only run one g, and it's not available.
// Make sure some other cpu is running to handle
// the ordinary run queue.
if(runtime_sched.gwait != 0) {
matchmg();
// m->lockedg might have been on the queue.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
}
} else {
// Look for work on global queue.
while(haveg() && canaddmcpu()) {
gp = gget();
if(gp == nil)
runtime_throw("gget inconsistency");
if(gp->lockedm) {
mnextg(gp->lockedm, gp);
continue;
}
runtime_sched.grunning++;
schedunlock();
return gp;
}
// The while loop ended either because the g queue is empty
// or because we have maxed out our m procs running go
// code (mcpu >= mcpumax). We need to check that
// concurrent actions by entersyscall/exitsyscall cannot
// invalidate the decision to end the loop.
//
// We hold the sched lock, so no one else is manipulating the
// g queue or changing mcpumax. Entersyscall can decrement
// mcpu, but if does so when there is something on the g queue,
// the gwait bit will be set, so entersyscall will take the slow path
// and use the sched lock. So it cannot invalidate our decision.
//
// Wait on global m queue.
mput(m);
}
v = runtime_atomicload(&runtime_sched.atomic);
if(runtime_sched.grunning == 0)
runtime_throw("all goroutines are asleep - deadlock!");
m->nextg = nil;
m->waitnextg = 1;
runtime_noteclear(&m->havenextg);
// Stoptheworld is waiting for all but its cpu to go to stop.
// Entersyscall might have decremented mcpu too, but if so
// it will see the waitstop and take the slow path.
// Exitsyscall never increments mcpu beyond mcpumax.
if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
// set waitstop = 0 (known to be 1)
runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
runtime_notewakeup(&runtime_sched.stopped);
}
schedunlock();
runtime_notesleep(&m->havenextg);
if(m->helpgc) {
runtime_gchelper();
m->helpgc = 0;
runtime_lock(&runtime_sched);
goto top;
}
if((gp = m->nextg) == nil)
runtime_throw("bad m->nextg in nextgoroutine");
m->nextg = nil;
return gp;
}