int32
runtime·helpgc(bool *extra)
{
M *mp;
int32 n, max;
// Figure out how many CPUs to use.
// Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
max = runtime·gomaxprocs;
if(max > runtime·ncpu)
max = runtime·ncpu;
if(max > MaxGcproc)
max = MaxGcproc;
// We're going to use one CPU no matter what.
// Figure out the max number of additional CPUs.
max--;
runtime·lock(&runtime·sched);
n = 0;
while(n < max && (mp = mget(nil)) != nil) {
n++;
mp->helpgc = 1;
mp->waitnextg = 0;
runtime·notewakeup(&mp->havenextg);
}
runtime·unlock(&runtime·sched);
if(extra)
*extra = n != max;
return n;
}
void
runtime·entersyscall(void)
{
if(runtime·sched.predawn)
return;
schedlock();
g->status = Gsyscall;
runtime·sched.mcpu--;
runtime·sched.msyscall++;
if(runtime·sched.gwait != 0)
matchmg();
if(runtime·sched.waitstop && runtime·sched.mcpu <= runtime·sched.mcpumax) {
runtime·sched.waitstop = 0;
runtime·notewakeup(&runtime·sched.stopped);
}
// Leave SP around for gc and traceback.
// Do before schedunlock so that gc
// never sees Gsyscall with wrong stack.
runtime·gosave(&g->sched);
g->gcsp = g->sched.sp;
g->gcstack = g->stackbase;
g->gcguard = g->stackguard;
if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) {
runtime·printf("entersyscall inconsistent %p [%p,%p]\n", g->gcsp, g->gcguard-StackGuard, g->gcstack);
runtime·throw("entersyscall");
}
schedunlock();
}
// Unlock the scheduler.
static void
schedunlock(void)
{
M *m;
m = mwakeup;
mwakeup = nil;
runtime·unlock(&runtime·sched);
if(m != nil)
runtime·notewakeup(&m->havenextg);
}
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
// Makes sure that at most $GOMAXPROCS gs are
// running on cpus (not in system calls) at any given time.
static G*
nextgandunlock(void)
{
G *gp;
if(runtime·sched.mcpu < 0)
runtime·throw("negative runtime·sched.mcpu");
// If there is a g waiting as m->nextg,
// mnextg took care of the runtime·sched.mcpu++.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
if(m->lockedg != nil) {
// We can only run one g, and it's not available.
// Make sure some other cpu is running to handle
// the ordinary run queue.
if(runtime·sched.gwait != 0)
matchmg();
} else {
// Look for work on global queue.
while(runtime·sched.mcpu < runtime·sched.mcpumax && (gp=gget()) != nil) {
if(gp->lockedm) {
mnextg(gp->lockedm, gp);
continue;
}
runtime·sched.mcpu++; // this m will run gp
schedunlock();
return gp;
}
// Otherwise, wait on global m queue.
mput(m);
}
if(runtime·sched.mcpu == 0 && runtime·sched.msyscall == 0)
runtime·throw("all goroutines are asleep - deadlock!");
m->nextg = nil;
m->waitnextg = 1;
runtime·noteclear(&m->havenextg);
if(runtime·sched.waitstop && runtime·sched.mcpu <= runtime·sched.mcpumax) {
runtime·sched.waitstop = 0;
runtime·notewakeup(&runtime·sched.stopped);
}
schedunlock();
runtime·notesleep(&m->havenextg);
if((gp = m->nextg) == nil)
runtime·throw("bad m->nextg in nextgoroutine");
m->nextg = nil;
return gp;
}
// Pass g to m for running.
// Caller has already incremented mcpu.
static void
mnextg(M *m, G *g)
{
runtime·sched.grunning++;
m->nextg = g;
if(m->waitnextg) {
m->waitnextg = 0;
if(mwakeup != nil)
runtime·notewakeup(&mwakeup->havenextg);
mwakeup = m;
}
}
// Pass g to m for running.
// Caller has already incremented mcpu.
static void
mnextg(M *mp, G *gp)
{
runtime·sched.grunning++;
mp->nextg = gp;
if(mp->waitnextg) {
mp->waitnextg = 0;
if(mwakeup != nil)
runtime·notewakeup(&mwakeup->havenextg);
mwakeup = mp;
}
}
void
runtime·entersyscall(void)
{
uint32 v;
if(m->profilehz > 0)
runtime·setprof(false);
// Leave SP around for gc and traceback.
runtime·gosave(&g->sched);
g->gcsp = g->sched.sp;
g->gcstack = g->stackbase;
g->gcguard = g->stackguard;
g->status = Gsyscall;
if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) {
// runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
// g->gcsp, g->gcguard-StackGuard, g->gcstack);
runtime·throw("entersyscall");
}
// Fast path.
// The slow path inside the schedlock/schedunlock will get
// through without stopping if it does:
// mcpu--
// gwait not true
// waitstop && mcpu <= mcpumax not true
// If we can do the same with a single atomic add,
// then we can skip the locks.
v = runtime·xadd(&runtime·sched.atomic, -1<<mcpuShift);
if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
return;
schedlock();
v = runtime·atomicload(&runtime·sched.atomic);
if(atomic_gwaiting(v)) {
matchmg();
v = runtime·atomicload(&runtime·sched.atomic);
}
if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
runtime·notewakeup(&runtime·sched.stopped);
}
// Re-save sched in case one of the calls
// (notewakeup, matchmg) triggered something using it.
runtime·gosave(&g->sched);
schedunlock();
}
void
runtime·helpgc(int32 nproc)
{
M *mp;
int32 n;
runtime·lock(&runtime·sched);
for(n = 1; n < nproc; n++) { // one M is currently running
mp = mget(nil);
if(mp == nil)
runtime·throw("runtime·gcprocs inconsistency");
mp->helpgc = 1;
mp->waitnextg = 0;
runtime·notewakeup(&mp->havenextg);
}
runtime·unlock(&runtime·sched);
}
static void
addtimer ( Timer *t )
{
int32 n;
Timer **nt;
#line 110 "/home/14/ren/source/golang/go/src/pkg/runtime/time.goc"
if ( t->when < 0 )
t->when = ( 1LL<<63 ) -1;
#line 113 "/home/14/ren/source/golang/go/src/pkg/runtime/time.goc"
if ( timers.len >= timers.cap ) {
#line 115 "/home/14/ren/source/golang/go/src/pkg/runtime/time.goc"
n = 16;
if ( n <= timers.cap )
n = timers.cap*3 / 2;
nt = runtime·malloc ( n*sizeof nt[0] ) ;
runtime·memmove ( nt , timers.t , timers.len*sizeof nt[0] ) ;
runtime·free ( timers.t ) ;
timers.t = nt;
timers.cap = n;
}
t->i = timers.len++;
timers.t[t->i] = t;
siftup ( t->i ) ;
if ( t->i == 0 ) {
#line 129 "/home/14/ren/source/golang/go/src/pkg/runtime/time.goc"
if ( timers.sleeping ) {
timers.sleeping = false;
runtime·notewakeup ( &timers.waitnote ) ;
}
if ( timers.rescheduling ) {
timers.rescheduling = false;
runtime·ready ( timers.timerproc ) ;
}
}
if ( timers.timerproc == nil ) {
timers.timerproc = runtime·newproc1 ( &timerprocv , nil , 0 , 0 , addtimer ) ;
timers.timerproc->issystem = true;
}
if ( debug )
dumptimers ( "addtimer" ) ;
}
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
// Makes sure that at most $GOMAXPROCS g's are
// running on cpus (not in system calls) at any given time.
static G*
nextgandunlock(void)
{
G *gp;
uint32 v;
top:
if(atomic_mcpu(runtime·sched.atomic) >= maxgomaxprocs)
runtime·throw("negative mcpu");
// If there is a g waiting as m->nextg, the mcpu++
// happened before it was passed to mnextg.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
if(m->lockedg != nil) {
// We can only run one g, and it's not available.
// Make sure some other cpu is running to handle
// the ordinary run queue.
if(runtime·sched.gwait != 0) {
matchmg();
// m->lockedg might have been on the queue.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
}
} else {
// Look for work on global queue.
while(haveg() && canaddmcpu()) {
gp = gget();
if(gp == nil)
runtime·throw("gget inconsistency");
if(gp->lockedm) {
mnextg(gp->lockedm, gp);
continue;
}
runtime·sched.grunning++;
schedunlock();
return gp;
}
// The while loop ended either because the g queue is empty
// or because we have maxed out our m procs running go
// code (mcpu >= mcpumax). We need to check that
// concurrent actions by entersyscall/exitsyscall cannot
// invalidate the decision to end the loop.
//
// We hold the sched lock, so no one else is manipulating the
// g queue or changing mcpumax. Entersyscall can decrement
// mcpu, but if does so when there is something on the g queue,
// the gwait bit will be set, so entersyscall will take the slow path
// and use the sched lock. So it cannot invalidate our decision.
//
// Wait on global m queue.
mput(m);
}
// Look for deadlock situation.
// There is a race with the scavenger that causes false negatives:
// if the scavenger is just starting, then we have
// scvg != nil && grunning == 0 && gwait == 0
// and we do not detect a deadlock. It is possible that we should
// add that case to the if statement here, but it is too close to Go 1
// to make such a subtle change. Instead, we work around the
// false negative in trivial programs by calling runtime.gosched
// from the main goroutine just before main.main.
// See runtime·main above.
//
// On a related note, it is also possible that the scvg == nil case is
// wrong and should include gwait, but that does not happen in
// standard Go programs, which all start the scavenger.
//
if((scvg == nil && runtime·sched.grunning == 0) ||
(scvg != nil && runtime·sched.grunning == 1 && runtime·sched.gwait == 0 &&
(scvg->status == Grunning || scvg->status == Gsyscall))) {
runtime·throw("all goroutines are asleep - deadlock!");
}
m->nextg = nil;
m->waitnextg = 1;
runtime·noteclear(&m->havenextg);
// Stoptheworld is waiting for all but its cpu to go to stop.
// Entersyscall might have decremented mcpu too, but if so
// it will see the waitstop and take the slow path.
// Exitsyscall never increments mcpu beyond mcpumax.
v = runtime·atomicload(&runtime·sched.atomic);
if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
// set waitstop = 0 (known to be 1)
runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
runtime·notewakeup(&runtime·sched.stopped);
}
schedunlock();
runtime·notesleep(&m->havenextg);
if(m->helpgc) {
runtime·gchelper();
m->helpgc = 0;
runtime·lock(&runtime·sched);
goto top;
}
if((gp = m->nextg) == nil)
runtime·throw("bad m->nextg in nextgoroutine");
m->nextg = nil;
return gp;
}
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
// Makes sure that at most $GOMAXPROCS g's are
// running on cpus (not in system calls) at any given time.
static G*
nextgandunlock(void)
{
G *gp;
uint32 v;
top:
if(atomic_mcpu(runtime·sched.atomic) >= maxgomaxprocs)
runtime·throw("negative mcpu");
// If there is a g waiting as m->nextg, the mcpu++
// happened before it was passed to mnextg.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
if(m->lockedg != nil) {
// We can only run one g, and it's not available.
// Make sure some other cpu is running to handle
// the ordinary run queue.
if(runtime·sched.gwait != 0) {
matchmg();
// m->lockedg might have been on the queue.
if(m->nextg != nil) {
gp = m->nextg;
m->nextg = nil;
schedunlock();
return gp;
}
}
} else {
// Look for work on global queue.
while(haveg() && canaddmcpu()) {
gp = gget();
if(gp == nil)
runtime·throw("gget inconsistency");
if(gp->lockedm) {
mnextg(gp->lockedm, gp);
continue;
}
runtime·sched.grunning++;
schedunlock();
return gp;
}
// The while loop ended either because the g queue is empty
// or because we have maxed out our m procs running go
// code (mcpu >= mcpumax). We need to check that
// concurrent actions by entersyscall/exitsyscall cannot
// invalidate the decision to end the loop.
//
// We hold the sched lock, so no one else is manipulating the
// g queue or changing mcpumax. Entersyscall can decrement
// mcpu, but if does so when there is something on the g queue,
// the gwait bit will be set, so entersyscall will take the slow path
// and use the sched lock. So it cannot invalidate our decision.
//
// Wait on global m queue.
mput(m);
}
// Look for deadlock situation: one single active g which happens to be scvg.
if(runtime·sched.grunning == 1 && runtime·sched.gwait == 0) {
if(scvg->status == Grunning || scvg->status == Gsyscall)
runtime·throw("all goroutines are asleep - deadlock!");
}
m->nextg = nil;
m->waitnextg = 1;
runtime·noteclear(&m->havenextg);
// Stoptheworld is waiting for all but its cpu to go to stop.
// Entersyscall might have decremented mcpu too, but if so
// it will see the waitstop and take the slow path.
// Exitsyscall never increments mcpu beyond mcpumax.
v = runtime·atomicload(&runtime·sched.atomic);
if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
// set waitstop = 0 (known to be 1)
runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
runtime·notewakeup(&runtime·sched.stopped);
}
schedunlock();
runtime·notesleep(&m->havenextg);
if(m->helpgc) {
runtime·gchelper();
m->helpgc = 0;
runtime·lock(&runtime·sched);
goto top;
}
if((gp = m->nextg) == nil)
runtime·throw("bad m->nextg in nextgoroutine");
m->nextg = nil;
return gp;
}
