/* get a new iop */
static
gasnete_iop_t *gasnete_iop_new(gasneti_threaddata_t * const thread) {
gasnete_iop_t *iop = thread->iop_free;
if_pt (iop) {
thread->iop_free = iop->next;
gasneti_memcheck(iop);
gasneti_assert(OPTYPE(iop) == OPTYPE_IMPLICIT);
gasneti_assert(iop->threadidx == thread->threadidx);
/* If using trace or stats, want meaningful counts when tracing NBI access regions */
#if GASNETI_STATS_OR_TRACE
iop->initiated_get_cnt = 0;
iop->initiated_put_cnt = 0;
gasnetc_atomic_set(&(iop->completed_get_cnt), 0, 0);
gasnetc_atomic_set(&(iop->completed_put_cnt), 0, 0);
#endif
} else {
extern void gasnete_amref_get_nbi_bulk (void *dest, gasnet_node_t node, void *src, size_t nbytes GASNETE_THREAD_FARG) {
gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD;
gasnete_iop_t * const op = mythread->current_iop;
GASNETI_CHECKPSHM_GET(UNALIGNED,V);
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
op->initiated_get_cnt++;
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_amref_get_reqh),
(gasnet_handlerarg_t)nbytes, PACK(dest), PACK(src), PACK_IOP_DONE(op,get))));
return;
} else {
int chunksz;
gasnet_handler_t reqhandler;
uint8_t *psrc = src;
uint8_t *pdest = dest;
#if GASNETE_USE_LONG_GETS
gasneti_memcheck(gasneti_seginfo);
if (gasneti_in_segment(gasneti_mynode, dest, nbytes)) {
chunksz = gasnet_AMMaxLongReply();
reqhandler = gasneti_handleridx(gasnete_amref_getlong_reqh);
}
else
#endif
{ reqhandler = gasneti_handleridx(gasnete_amref_get_reqh);
chunksz = gasnet_AMMaxMedium();
}
for (;;) {
op->initiated_get_cnt++;
if (nbytes > chunksz) {
GASNETI_SAFE(
SHORT_REQ(4,7,(node, reqhandler,
(gasnet_handlerarg_t)chunksz, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get))));
nbytes -= chunksz;
psrc += chunksz;
pdest += chunksz;
} else {
GASNETI_SAFE(
SHORT_REQ(4,7,(node, reqhandler,
(gasnet_handlerarg_t)nbytes, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get))));
break;
}
}
return;
}
}
/* allocate more eops */
GASNETI_NEVER_INLINE(gasnete_eop_alloc,
static void gasnete_eop_alloc(gasneti_threaddata_t * const thread)) {
gasnete_eopaddr_t addr;
int bufidx = thread->eop_num_bufs;
gasnete_eop_t *buf;
int i;
gasnete_threadidx_t threadidx = thread->threadidx;
if (bufidx == 256) gasneti_fatalerror("GASNet Extended API: Ran out of explicit handles (limit=65535)");
thread->eop_num_bufs++;
buf = (gasnete_eop_t *)gasneti_calloc(256,sizeof(gasnete_eop_t));
gasneti_leak(buf);
for (i=0; i < 256; i++) {
addr.bufferidx = bufidx;
#if GASNETE_SCATTER_EOPS_ACROSS_CACHELINES
#ifdef GASNETE_EOP_MOD
addr.eopidx = (i+32) % 255;
#else
{ int k = i+32;
addr.eopidx = k > 255 ? k - 255 : k;
}
#endif
#else
addr.eopidx = i+1;
#endif
buf[i].threadidx = threadidx;
buf[i].addr = addr;
#if 0 /* these can safely be skipped when the values are zero */
SET_OPSTATE(&(buf[i]),OPSTATE_FREE);
SET_OPTYPE(&(buf[i]),OPTYPE_EXPLICIT);
#if GASNETE_EOP_COUNTED
buff[i].initiated_cnt = 0;
#endif
#endif
#if GASNETE_EOP_COUNTED
gasnetc_atomic_set(&(buf[i].completed_cnt), 0, 0);
#endif
}
/* add a list terminator */
#if GASNETE_SCATTER_EOPS_ACROSS_CACHELINES
#ifdef GASNETE_EOP_MOD
buf[223].addr.eopidx = 255; /* modular arithmetic messes up this one */
#endif
buf[255].addr = EOPADDR_NIL;
#else
buf[255].addr = EOPADDR_NIL;
#endif
thread->eop_bufs[bufidx] = buf;
addr.bufferidx = bufidx;
addr.eopidx = 0;
thread->eop_free = addr;
#if GASNET_DEBUG
{ /* verify new free list got built correctly */
int i;
int seen[256];
gasnete_eopaddr_t addr = thread->eop_free;
#if 0
if (gasneti_mynode == 0)
for (i=0;i<256;i++) {
fprintf(stderr,"%i: %i: next=%i\n",gasneti_mynode,i,buf[i].addr.eopidx);
fflush(stderr);
}
sleep(5);
#endif
gasneti_memcheck(thread->eop_bufs[bufidx]);
memset(seen, 0, 256*sizeof(int));
for (i=0;i<(bufidx==255?255:256);i++) {
gasnete_eop_t *eop;
gasneti_assert(!gasnete_eopaddr_isnil(addr));
eop = GASNETE_EOPADDR_TO_PTR(thread,addr);
gasneti_assert(OPTYPE(eop) == OPTYPE_EXPLICIT);
gasneti_assert(OPSTATE(eop) == OPSTATE_FREE);
gasneti_assert(eop->threadidx == threadidx);
gasneti_assert(addr.bufferidx == bufidx);
gasneti_assert(!seen[addr.eopidx]);/* see if we hit a cycle */
seen[addr.eopidx] = 1;
addr = eop->addr;
}
gasneti_assert(gasnete_eopaddr_isnil(addr));
}
#endif
}
