static void
RecordSpan(void *vh, byte *p)
{
MHeap *h;
MSpan *s;
MSpan **all;
uint32 cap;
h = vh;
s = (MSpan*)p;
if(h->nspan >= h->nspancap) {
cap = 64*1024/sizeof(all[0]);
if(cap < h->nspancap*3/2)
cap = h->nspancap*3/2;
all = (MSpan**)runtime_SysAlloc(cap*sizeof(all[0]), &mstats.other_sys);
if(all == nil)
runtime_throw("runtime: cannot allocate memory");
if(h->allspans) {
runtime_memmove(all, h->allspans, h->nspancap*sizeof(all[0]));
// Don't free the old array if it's referenced by sweep.
// See the comment in mgc0.c.
if(h->allspans != runtime_mheap.sweepspans)
runtime_SysFree(h->allspans, h->nspancap*sizeof(all[0]), &mstats.other_sys);
}
h->allspans = all;
h->nspancap = cap;
}
h->allspans[h->nspan++] = s;
}
static void
RecordSpan(void *vh, byte *p)
{
MHeap *h;
MSpan *s;
MSpan **all;
uint32 cap;
h = vh;
s = (MSpan*)p;
if(h->nspan >= h->nspancap) {
cap = 64*1024/sizeof(all[0]);
if(cap < h->nspancap*3/2)
cap = h->nspancap*3/2;
all = (MSpan**)runtime_SysAlloc(cap*sizeof(all[0]));
if(all == nil)
runtime_throw("runtime: cannot allocate memory");
if(h->allspans) {
runtime_memmove(all, h->allspans, h->nspancap*sizeof(all[0]));
runtime_SysFree(h->allspans, h->nspancap*sizeof(all[0]));
}
h->allspans = all;
h->nspancap = cap;
}
h->allspans[h->nspan++] = s;
}
static void
hwrite(const byte *data, uintptr len)
{
if(len + nbuf <= BufSize) {
runtime_memmove(buf + nbuf, data, len);
nbuf += len;
return;
}
runtime_write(dumpfd, buf, nbuf);
if(len >= BufSize) {
runtime_write(dumpfd, data, len);
nbuf = 0;
} else {
runtime_memmove(buf, data, len);
nbuf = len;
}
}
// write to goroutine-local buffer if diverting output,
// or else standard error.
static void
gwrite(const void *v, intgo n)
{
G* g = runtime_g();
if(g == nil || g->writebuf == nil) {
// Avoid -D_FORTIFY_SOURCE problems.
int rv __attribute__((unused));
rv = runtime_write(2, v, n);
return;
}
if(g->writenbuf == 0)
return;
if(n > g->writenbuf)
n = g->writenbuf;
runtime_memmove(g->writebuf, v, n);
g->writebuf += n;
g->writenbuf -= n;
}
static bool
chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received)
{
SudoG *sg;
SudoG mysg;
G *gp;
int64 t0;
G *g;
if(runtime_gcwaiting())
runtime_gosched();
// raceenabled: don't need to check ep, as it is always on the stack.
if(debug)
runtime_printf("chanrecv: chan=%p\n", c);
g = runtime_g();
if(c == nil) {
USED(t);
if(!block)
return false;
runtime_park(nil, nil, "chan receive (nil chan)");
return false; // not reached
}
t0 = 0;
mysg.releasetime = 0;
if(runtime_blockprofilerate > 0) {
t0 = runtime_cputicks();
mysg.releasetime = -1;
}
runtime_lock(c);
if(c->dataqsiz > 0)
goto asynch;
if(c->closed)
goto closed;
sg = dequeue(&c->sendq);
if(sg != nil) {
if(raceenabled)
racesync(c, sg);
runtime_unlock(c);
if(ep != nil)
runtime_memmove(ep, sg->elem, c->elemsize);
gp = sg->g;
gp->param = sg;
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
if(received != nil)
*received = true;
return true;
}
if(!block) {
runtime_unlock(c);
return false;
}
mysg.elem = ep;
mysg.g = g;
mysg.selectdone = nil;
g->param = nil;
enqueue(&c->recvq, &mysg);
runtime_parkunlock(c, "chan receive");
if(g->param == nil) {
runtime_lock(c);
if(!c->closed)
runtime_throw("chanrecv: spurious wakeup");
goto closed;
}
if(received != nil)
*received = true;
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return true;
asynch:
if(c->qcount <= 0) {
if(c->closed)
goto closed;
if(!block) {
runtime_unlock(c);
if(received != nil)
*received = false;
return false;
}
mysg.g = g;
mysg.elem = nil;
mysg.selectdone = nil;
enqueue(&c->recvq, &mysg);
runtime_parkunlock(c, "chan receive");
runtime_lock(c);
goto asynch;
}
if(raceenabled)
runtime_raceacquire(chanbuf(c, c->recvx));
if(ep != nil)
runtime_memmove(ep, chanbuf(c, c->recvx), c->elemsize);
runtime_memclr(chanbuf(c, c->recvx), c->elemsize);
if(++c->recvx == c->dataqsiz)
c->recvx = 0;
c->qcount--;
sg = dequeue(&c->sendq);
if(sg != nil) {
gp = sg->g;
runtime_unlock(c);
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
} else
runtime_unlock(c);
if(received != nil)
*received = true;
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return true;
closed:
if(ep != nil)
runtime_memclr(ep, c->elemsize);
if(received != nil)
*received = false;
if(raceenabled)
runtime_raceacquire(c);
runtime_unlock(c);
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return true;
}
/*
* generic single channel send/recv
* if the bool pointer is nil,
* then the full exchange will
* occur. if pres is not nil,
* then the protocol will not
* sleep but return if it could
* not complete.
*
* sleep can wake up with g->param == nil
* when a channel involved in the sleep has
* been closed. it is easiest to loop and re-run
* the operation; we'll see that it's now closed.
*/
static bool
chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc)
{
SudoG *sg;
SudoG mysg;
G* gp;
int64 t0;
G* g;
g = runtime_g();
if(raceenabled)
runtime_racereadobjectpc(ep, t->__element_type, runtime_getcallerpc(&t), chansend);
if(c == nil) {
USED(t);
if(!block)
return false;
runtime_park(nil, nil, "chan send (nil chan)");
return false; // not reached
}
if(runtime_gcwaiting())
runtime_gosched();
if(debug) {
runtime_printf("chansend: chan=%p\n", c);
}
t0 = 0;
mysg.releasetime = 0;
if(runtime_blockprofilerate > 0) {
t0 = runtime_cputicks();
mysg.releasetime = -1;
}
runtime_lock(c);
if(raceenabled)
runtime_racereadpc(c, pc, chansend);
if(c->closed)
goto closed;
if(c->dataqsiz > 0)
goto asynch;
sg = dequeue(&c->recvq);
if(sg != nil) {
if(raceenabled)
racesync(c, sg);
runtime_unlock(c);
gp = sg->g;
gp->param = sg;
if(sg->elem != nil)
runtime_memmove(sg->elem, ep, c->elemsize);
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
return true;
}
if(!block) {
runtime_unlock(c);
return false;
}
mysg.elem = ep;
mysg.g = g;
mysg.selectdone = nil;
g->param = nil;
enqueue(&c->sendq, &mysg);
runtime_parkunlock(c, "chan send");
if(g->param == nil) {
runtime_lock(c);
if(!c->closed)
runtime_throw("chansend: spurious wakeup");
goto closed;
}
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return true;
asynch:
if(c->closed)
goto closed;
if(c->qcount >= c->dataqsiz) {
if(!block) {
runtime_unlock(c);
return false;
}
mysg.g = g;
mysg.elem = nil;
mysg.selectdone = nil;
enqueue(&c->sendq, &mysg);
runtime_parkunlock(c, "chan send");
runtime_lock(c);
goto asynch;
}
if(raceenabled)
runtime_racerelease(chanbuf(c, c->sendx));
runtime_memmove(chanbuf(c, c->sendx), ep, c->elemsize);
if(++c->sendx == c->dataqsiz)
c->sendx = 0;
c->qcount++;
sg = dequeue(&c->recvq);
if(sg != nil) {
gp = sg->g;
runtime_unlock(c);
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
} else
runtime_unlock(c);
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return true;
closed:
runtime_unlock(c);
runtime_panicstring("send on closed channel");
return false; // not reached
}
/*
* generic single channel send/recv
* if the bool pointer is nil,
* then the full exchange will
* occur. if pres is not nil,
* then the protocol will not
* sleep but return if it could
* not complete.
*
* sleep can wake up with g->param == nil
* when a channel involved in the sleep has
* been closed. it is easiest to loop and re-run
* the operation; we'll see that it's now closed.
*/
void
runtime_chansend(ChanType *t, Hchan *c, byte *ep, bool *pres, void *pc)
{
SudoG *sg;
SudoG mysg;
G* gp;
int64 t0;
G* g;
g = runtime_g();
if(c == nil) {
USED(t);
if(pres != nil) {
*pres = false;
return;
}
runtime_park(nil, nil, "chan send (nil chan)");
return; // not reached
}
if(runtime_gcwaiting)
runtime_gosched();
if(debug) {
runtime_printf("chansend: chan=%p\n", c);
}
t0 = 0;
mysg.releasetime = 0;
if(runtime_blockprofilerate > 0) {
t0 = runtime_cputicks();
mysg.releasetime = -1;
}
runtime_lock(c);
// TODO(dvyukov): add similar instrumentation to select.
if(raceenabled)
runtime_racereadpc(c, pc);
if(c->closed)
goto closed;
if(c->dataqsiz > 0)
goto asynch;
sg = dequeue(&c->recvq);
if(sg != nil) {
if(raceenabled)
racesync(c, sg);
runtime_unlock(c);
gp = sg->g;
gp->param = sg;
if(sg->elem != nil)
runtime_memmove(sg->elem, ep, c->elemsize);
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
if(pres != nil)
*pres = true;
return;
}
if(pres != nil) {
runtime_unlock(c);
*pres = false;
return;
}
mysg.elem = ep;
mysg.g = g;
mysg.selgen = NOSELGEN;
g->param = nil;
enqueue(&c->sendq, &mysg);
runtime_park(runtime_unlock, c, "chan send");
if(g->param == nil) {
runtime_lock(c);
if(!c->closed)
runtime_throw("chansend: spurious wakeup");
goto closed;
}
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return;
asynch:
if(c->closed)
goto closed;
if(c->qcount >= c->dataqsiz) {
if(pres != nil) {
runtime_unlock(c);
*pres = false;
return;
}
mysg.g = g;
mysg.elem = nil;
mysg.selgen = NOSELGEN;
enqueue(&c->sendq, &mysg);
runtime_park(runtime_unlock, c, "chan send");
runtime_lock(c);
goto asynch;
}
if(raceenabled)
runtime_racerelease(chanbuf(c, c->sendx));
runtime_memmove(chanbuf(c, c->sendx), ep, c->elemsize);
if(++c->sendx == c->dataqsiz)
c->sendx = 0;
c->qcount++;
sg = dequeue(&c->recvq);
if(sg != nil) {
gp = sg->g;
runtime_unlock(c);
if(sg->releasetime)
sg->releasetime = runtime_cputicks();
runtime_ready(gp);
} else
runtime_unlock(c);
if(pres != nil)
*pres = true;
if(mysg.releasetime > 0)
runtime_blockevent(mysg.releasetime - t0, 2);
return;
closed:
runtime_unlock(c);
runtime_panicstring("send on closed channel");
}