void gasnetc_bootstrapBroadcast(void *src, size_t len, void *dest, int rootnode) {
int retval;
gasneti_assert(gasneti_nodes > 0 && gasneti_mynode < gasneti_nodes);
if (gasneti_mynode == rootnode) memcpy(dest, src, len);
GASNETI_AM_SAFE_NORETURN(retval,AMMPI_SPMDBroadcast(dest, len, rootnode));
if_pf (retval) gasneti_fatalerror("failure in gasnetc_bootstrapBroadcast()");
}
int gasnetc_AM_Generic(gasnetc_category_t category,
int dest,
gasnet_handler_t handler_id,
void *src_addr,
int nbytes,
void *dst_addr,
uint8_t is_request,
uint8_t is_sync_request,
uint8_t msg_num,
int numargs, va_list argptr)
{
gasnet_mxm_send_req_t sreq;
gasnet_mxm_send_req_t * p_sreq;
mxm_send_req_t * mxm_sreq;
int i, sge_idx = 0;
/*
* Encoding the following metadata in the immediate field:
* - AM handler ID (1 byte)
* - number of args (1 byte)
* - message category (2 bits)
* - whether the message is a request (1 bit)
* - whether the message is a synchronous request (1 bit)
* - message number (1 byte)
*/
mxm_imm_t imm_data = (mxm_imm_t) GASNETC_MSG_METADATA(
handler_id, numargs, category,
is_request, is_sync_request,
SYSTEM_MSG_NA, msg_num);
/* using source node ID as tag */
mxm_tag_t tag = (mxm_tag_t) gasnet_mynode();
if_pf (GASNETC_LONG_ASYNC_REQ(category, is_request, is_sync_request))
p_sreq = gasnetc_alloc_send_req();
else
extern int gasnetc_AMPoll(void) {
int retval;
GASNETI_CHECKATTACH();
#if GASNET_PSHM
gasneti_AMPSHMPoll(0);
#endif
AMLOCK();
GASNETI_AM_SAFE_NORETURN(retval,AM_Poll(gasnetc_bundle));
AMUNLOCK();
if_pf (retval) GASNETI_RETURN_ERR(RESOURCE);
else return GASNET_OK;
extern int
firehose_move_callback(gasnet_node_t node,
const firehose_region_t *unpin_list,
size_t unpin_num,
firehose_region_t *pin_list,
size_t pin_num)
{
GASNETC_TRACE_WAIT_BEGIN();
int rc;
int h, i;
const enum ibv_access_flags access = (enum ibv_access_flags)
(IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ);
/* Take care of any unpins first */
for (i = 0; i < unpin_num; i++) {
GASNETC_FOR_ALL_HCA_INDEX(h) {
rc = ibv_dereg_mr(unpin_list[i].client.handle[h]);
GASNETC_IBV_CHECK(rc, "from ibv_dereg_mr");
}
GASNETC_TRACE_UNPIN(&unpin_list[i]);
}
/* Take care of any pins */
for (i = 0; i < pin_num; i++) {
firehose_region_t *region = pin_list + i;
firehose_client_t *client = ®ion->client;
gasneti_assert(region->addr % GASNET_PAGESIZE == 0);
gasneti_assert(region->len % GASNET_PAGESIZE == 0);
GASNETC_FOR_ALL_HCA_INDEX(h) {
client->handle[h] = ibv_reg_mr(gasnetc_hca[h].pd, (void*)(uintptr_t)region->addr, region->len, access);
if_pf (client->handle[h] == NULL)
gasneti_fatalerror("ibv_reg_mr failed in firehose_move_callback errno=%d (%s)", errno, strerror(errno));
client->lkey[h] = client->handle[h]->lkey;
client->rkey[h] = client->handle[h]->rkey;
}
GASNETC_TRACE_PIN(&pin_list[i]);
}
GASNETC_TRACE_WAIT_END(FIREHOSE_MOVE);
return 0;
}
extern int gasnetc_AMPoll(GASNETI_THREAD_FARG_ALONE) {
int retval;
GASNETI_CHECKATTACH();
#if GASNET_PSHM
gasneti_AMPSHMPoll(0 GASNETI_THREAD_PASS);
#endif
AMLOCK();
// In single-nbrhd case never need to poll the network for client AMs.
// However, we'll still check for control traffic for orderly exit handling.
if (gasneti_mysupernode.grp_count > 1) {
GASNETI_AM_SAFE_NORETURN(retval,AM_Poll(gasnetc_bundle));
} else {
// TODO-EX: a lock-free peek would allow elimination of a lock cycle
GASNETI_AM_SAFE_NORETURN(retval,AMUDP_SPMDHandleControlTraffic(NULL));
}
AMUNLOCK();
if_pf (retval) GASNETI_RETURN_ERR(RESOURCE);
else return GASNET_OK;
/*
Misc. Active Message Functions
==============================
*/
extern int gasnetc_AMGetMsgSource(gasnet_token_t token, gasnet_node_t *srcindex) {
int retval;
gasnet_node_t sourceid;
GASNETI_CHECKATTACH();
GASNETI_CHECK_ERRR((!token),BAD_ARG,"bad token");
GASNETI_CHECK_ERRR((!srcindex),BAD_ARG,"bad src ptr");
#if GASNET_PSHM
if (gasneti_AMPSHMGetMsgSource(token, &sourceid) != GASNET_OK)
#endif
{
int tmp; /* AMUDP wants an int, but gasnet_node_t is uint16_t */
GASNETI_AM_SAFE_NORETURN(retval,AMUDP_GetSourceId(token, &tmp));
if_pf (retval) GASNETI_RETURN_ERR(RESOURCE);
gasneti_assert(tmp >= 0);
sourceid = tmp;
}
gasneti_assert(sourceid < gasneti_nodes);
*srcindex = sourceid;
return GASNET_OK;
}
static inline void gasnetc_PostSend(mxm_send_req_t * mxm_sreq,
uint8_t is_request,
uint8_t block,
uint8_t msg_num)
{
mxm_error_t mxm_res;
#if GASNET_DEBUG_AM
MXM_LOG("[pid %d] calling mxm_req_send()...\n", getpid());
#endif
mxm_res = mxm_req_send(mxm_sreq);
#if GASNET_DEBUG_AM
MXM_LOG("[pid %d] mxm_req_send() returned %d\n", getpid(), (int)mxm_res);
#endif
if_pf (mxm_res != MXM_OK)
gasneti_fatalerror("Error posting send request - %s\n",
mxm_error_string(mxm_res));
if_pt (block)
gasnetc_ReqRepWait(mxm_sreq, is_request, msg_num);
}
void gasnetc_bootstrapExchange(void *src, size_t len, void *dest) {
int retval;
GASNETI_AM_SAFE_NORETURN(retval,AMUDP_SPMDAllGather(src, dest, len));
if_pf (retval) gasneti_fatalerror("failure in gasnetc_bootstrapExchange()");
}
void gasnetc_bootstrapBarrier(void) {
int retval;
AM_ASSERT_LOCKED(); /* need this because SPMDBarrier may poll */
GASNETI_AM_SAFE_NORETURN(retval,AMUDP_SPMDBarrier());
if_pf (retval) gasneti_fatalerror("failure in gasnetc_bootstrapBarrier()");
}
/* Packetizes remotelist into a list of gasnete_packetdesc_t entries based on maxpayload packet size
sharedpacket => metadata and corresponding data travel together in unified packets (put)
so that for each packet i: datasz_i + metadatasz_i <= maxpayload
!sharedpacket => metadata and corresponding data travel in separate packets (get)
so that for each packet i: MAX(datasz_i,metadatasz_i) <= maxpayload
A local packet table is also computed to match the remote packetization boundaries of the data
on a byte-for-byte basis
Allocates and populates the plocalpt and premotept arrays with the packetization information
Returns the number of packets described by the resulting plocalpt and premotept arrays
*/
size_t gasnete_packetize_addrlist(size_t remotecount, size_t remotelen,
size_t localcount, size_t locallen,
gasnete_packetdesc_t **premotept,
gasnete_packetdesc_t **plocalpt,
size_t maxpayload, int sharedpacket) {
size_t ptidx;
int done = 0;
size_t ridx = 0, roffset = 0, lidx = 0, loffset = 0;
size_t const metadatasz = sizeof(void *);
size_t const runit = (sharedpacket ? metadatasz + remotelen : MAX(metadatasz,remotelen));
size_t ptsz = (runit <= maxpayload ? /* conservative upper bound on packet count */
remotecount / (maxpayload / runit) + 1 :
remotelen*remotecount / (maxpayload - 2*metadatasz) + 1);
gasnete_packetdesc_t *remotept = gasneti_malloc(ptsz*sizeof(gasnete_packetdesc_t));
gasnete_packetdesc_t *localpt = gasneti_malloc(ptsz*sizeof(gasnete_packetdesc_t));
gasneti_assert(premotept && plocalpt && remotecount && remotelen && localcount && locallen);
gasneti_assert(remotecount*remotelen == localcount*locallen);
gasneti_assert(remotecount*remotelen > 0);
for (ptidx = 0; ; ptidx++) {
ssize_t packetremain = maxpayload;
ssize_t packetdata = 0;
size_t rdatasz, ldatasz;
gasneti_assert(ptidx < ptsz);
/* begin remote packet */
remotept[ptidx].firstidx = ridx;
remotept[ptidx].firstoffset = roffset;
/* begin local packet */
if_pf (lidx == localcount) localpt[ptidx].firstidx = lidx-1;
else localpt[ptidx].firstidx = lidx;
localpt[ptidx].firstoffset = loffset;
if (roffset > 0) { /* initial partial entry */
gasneti_assert(roffset < remotelen);
rdatasz = remotelen - roffset; /* data left in current entry */
/* try to add the entire entry to packet */
if (sharedpacket) packetremain -= (metadatasz + rdatasz);
else packetremain -= MAX(metadatasz, rdatasz);
if (packetremain < 0) { /* overflowed - finished a packet, and spill to next */
rdatasz += packetremain; /* compute truncated datasz that fits in this packet */
roffset += rdatasz; /* update offset into current entry */
packetdata += rdatasz;
goto rend;
} else {
packetdata += rdatasz;
roffset = 0; /* finished an entry */
ridx++;
if (ridx == remotecount) { done = 1; goto rend; } /* done - this is last packet */
}
}
if (packetremain >= runit) { /* whole entries */
size_t numunits = packetremain / runit;
if (ridx + numunits > remotecount) numunits = remotecount - ridx;
rdatasz = remotelen;
packetremain -= runit*numunits;
packetdata += remotelen*numunits;
ridx += numunits;
gasneti_assert(roffset == 0);
if (ridx == remotecount) { done = 1; goto rend; } /* done - this is last packet */
}
if (packetremain > metadatasz) { /* trailing partial entry */
gasneti_assert(packetremain < runit);
if (sharedpacket) rdatasz = packetremain - metadatasz;
else rdatasz = packetremain;
packetdata += rdatasz;
roffset = rdatasz;
}
rend:
/* end remote packet */
if (roffset == 0) remotept[ptidx].lastidx = ridx-1;
else remotept[ptidx].lastidx = ridx;
remotept[ptidx].lastlen = rdatasz;
#if GASNET_DEBUG /* verify packing properties */
gasnete_packetize_verify(remotept, ptidx, done, remotecount, remotelen, 0);
{ size_t datachk = 0, i;
size_t entries = remotept[ptidx].lastidx - remotept[ptidx].firstidx + 1;
for (i = remotept[ptidx].firstidx; i <= remotept[ptidx].lastidx; i++) {
if (i == remotept[ptidx].lastidx) datachk += remotept[ptidx].lastlen;
else if (i == remotept[ptidx].firstidx) datachk += (remotelen - remotept[ptidx].firstoffset);
else datachk += remotelen;
}
gasneti_assert(packetdata == datachk);
if (sharedpacket) {
gasneti_assert((metadatasz*entries + packetdata) <= maxpayload); /* not overfull */
gasneti_assert(((metadatasz*entries + packetdata) >= maxpayload - metadatasz) || done); /* not underfull */
} else {
gasneti_assert(MAX(metadatasz*entries,packetdata) <= maxpayload); /* not overfull */
gasneti_assert((MAX(metadatasz*entries,packetdata) >= maxpayload - 2*metadatasz) || done); /* not underfull */
}
}
#endif
ldatasz = 0;
if (loffset > 0) { /* initial partial entry */
gasneti_assert(loffset < locallen);
ldatasz = locallen - loffset; /* data left in current entry */
packetdata -= ldatasz;
if (packetdata < 0) { /* overflowed - this entry spills into next packet */
ldatasz += packetdata; /* compute truncated datasz that fits in this packet */
loffset += ldatasz; /* update offset into current entry */
packetdata = 0;
} else {
loffset = 0; /* finished an entry */
lidx++;
gasneti_assert(lidx < localcount || (lidx == localcount && packetdata == 0));
}
}
if (packetdata >= locallen) { /* whole entries */
size_t numunits = packetdata / locallen;
if (lidx + numunits > localcount) numunits = localcount - lidx;
ldatasz = locallen;
packetdata -= locallen*numunits;
lidx += numunits;
gasneti_assert(lidx < localcount || (lidx == localcount && packetdata == 0));
gasneti_assert(loffset == 0);
}
if (packetdata > 0) { /* trailing partial entry */
gasneti_assert(packetdata < locallen);
ldatasz = packetdata;
loffset = ldatasz;
}
/* end local packet */
if (loffset == 0) localpt[ptidx].lastidx = lidx-1;
else localpt[ptidx].lastidx = lidx;
localpt[ptidx].lastlen = ldatasz;
#if GASNET_DEBUG /* verify packing properties */
gasnete_packetize_verify(localpt, ptidx, done, localcount, locallen, 0);
#endif
if (done) {
gasneti_assert(ridx == remotecount && roffset == 0 && lidx == localcount && loffset == 0);
*premotept = remotept;
*plocalpt = localpt;
return ptidx+1;
}
}
/* Packetizes remotelist into a list of gasnete_packetdesc_t entries based on maxpayload packet size
sharedpacket => metadata and corresponding data travel together in unified packets (put)
so that for each packet i: datasz_i + metadatasz_i <= maxpayload
!sharedpacket => metadata and corresponding data travel in separate packets (get)
so that for each packet i: MAX(datasz_i,metadatasz_i) <= maxpayload
A local packet table is also computed to match the remote packetization boundaries of the data
on a byte-for-byte basis
Allocates and populates the plocalpt and premotept arrays with the packetization information
Returns the number of packets described by the resulting plocalpt and premotept arrays
*/
size_t gasnete_packetize_memvec(size_t remotecount, gasnet_memvec_t const remotelist[],
size_t localcount, gasnet_memvec_t const locallist[],
gasnete_packetdesc_t **premotept,
gasnete_packetdesc_t **plocalpt,
size_t maxpayload, int sharedpacket) {
size_t ptidx;
int done = 0;
size_t ridx = 0, roffset = 0, lidx = 0, loffset = 0;
size_t const metadatasz = sizeof(gasnet_memvec_t);
size_t ptsz = 4; /* initial size guess - no fast way to know for sure */
gasnete_packetdesc_t *remotept = gasneti_malloc(ptsz*sizeof(gasnete_packetdesc_t));
gasnete_packetdesc_t *localpt = gasneti_malloc(ptsz*sizeof(gasnete_packetdesc_t));
gasneti_assert(premotept && plocalpt && remotecount && localcount);
gasneti_assert(gasnete_memveclist_totalsz(remotecount,remotelist) ==
gasnete_memveclist_totalsz(localcount,locallist));
for (ptidx = 0; ; ptidx++) {
ssize_t packetremain = maxpayload;
ssize_t packetdata = 0;
size_t rdatasz, ldatasz;
if (ptidx == ptsz) { /* grow the packet tables */
ptsz *= 2;
remotept = gasneti_realloc(remotept, ptsz*sizeof(gasnete_packetdesc_t));
localpt = gasneti_realloc(localpt, ptsz*sizeof(gasnete_packetdesc_t));
}
/* begin remote packet */
remotept[ptidx].firstidx = ridx;
remotept[ptidx].firstoffset = roffset;
/* begin local packet */
if_pf (lidx == localcount) localpt[ptidx].firstidx = lidx-1; /* might happen if remote has trailing empties */
else localpt[ptidx].firstidx = lidx;
localpt[ptidx].firstoffset = loffset;
while (packetremain > metadatasz) { /* room for more entries */
gasneti_assert(roffset < remotelist[ridx].len || (remotelist[ridx].len == 0 && roffset == 0));
rdatasz = remotelist[ridx].len - roffset; /* data left in current entry */
/* try to add the entire entry to packet */
if (sharedpacket) packetremain -= (metadatasz + rdatasz);
else packetremain -= MAX(metadatasz, rdatasz);
if (packetremain < 0) { /* overflowed - finished a packet, and spill to next */
rdatasz += packetremain; /* compute truncated datasz that fits in this packet */
roffset += rdatasz; /* update offset into current entry */
packetdata += rdatasz;
break;
} else {
packetdata += rdatasz;
roffset = 0; /* finished an entry */
ridx++;
if (ridx == remotecount) { done = 1; break; } /* done - this is last packet */
}
}
/* end remote packet */
if (roffset == 0) remotept[ptidx].lastidx = ridx-1;
else remotept[ptidx].lastidx = ridx;
remotept[ptidx].lastlen = rdatasz;
#if GASNET_DEBUG /* verify packing properties */
gasnete_packetize_verify(remotept, ptidx, done, remotecount, 0, remotelist);
{ size_t datachk = 0, i;
size_t entries = remotept[ptidx].lastidx - remotept[ptidx].firstidx + 1;
for (i = remotept[ptidx].firstidx; i <= remotept[ptidx].lastidx; i++) {
if (i == remotept[ptidx].lastidx) datachk += remotept[ptidx].lastlen;
else if (i == remotept[ptidx].firstidx) datachk += (remotelist[i].len - remotept[ptidx].firstoffset);
else datachk += remotelist[i].len;
}
gasneti_assert(packetdata == datachk);
if (sharedpacket) {
gasneti_assert((metadatasz*entries + packetdata) <= maxpayload); /* not overfull */
gasneti_assert(((metadatasz*entries + packetdata) >= maxpayload - metadatasz) || done); /* not underfull */
} else {
gasneti_assert(MAX(metadatasz*entries,packetdata) <= maxpayload); /* not overfull */
/* algorithm currently may underfill for !sharedpacket, because it effectively always
subtracts the MAX(metadatasz, datasz) from *both* packets being managed simultaneously in packetremain,
rather than maintaining independent packetremains and updating each accordingly (increasing arithmetic complexity)
In vectors whose entries are dominated by datasz or metadatasz, the effect should be neglible
In perverse cases we might end up with a packet which where the maximal packet is only 2/3 full
this means in datasz dominated vectors with a few entries where datasz < metadatasz (or vice-versa)
*/
gasneti_assert((MAX(metadatasz*entries,packetdata) >= (maxpayload - metadatasz)/2) || done); /* not underfull */
}
}
#endif
ldatasz = 0;
while (packetdata > 0 || (lidx < localcount && locallist[lidx].len == 0)) {
gasneti_assert(loffset < locallist[lidx].len || (locallist[lidx].len == 0 && loffset == 0));
ldatasz = locallist[lidx].len - loffset; /* data left in current entry */
packetdata -= ldatasz;
if (packetdata < 0) { /* overflowed - this entry spills into next packet */
ldatasz += packetdata; /* compute truncated datasz that fits in this packet */
loffset += ldatasz; /* update offset into current entry */
break;
} else {
loffset = 0; /* finished an entry */
lidx++;
}
}
/* end local packet */
if (loffset == 0) localpt[ptidx].lastidx = lidx-1;
else localpt[ptidx].lastidx = lidx;
localpt[ptidx].lastlen = ldatasz;
#if GASNET_DEBUG /* verify packing properties */
gasnete_packetize_verify(localpt, ptidx, done, localcount, 0, locallist);
#endif
if (done) {
gasneti_assert(ridx == remotecount && roffset == 0 && lidx == localcount && loffset == 0);
*premotept = remotept;
*plocalpt = localpt;
return ptidx+1;
}
}
/* ##################################################################### */
void
gasnete_fh_callback_put(struct gm_port *p, void *context,
gm_status_t status) {
#ifndef GASNET_SEGMENT_FAST
GASNET_BEGIN_FUNCTION(); /* thread cache for *_IN_UNKNOWN */
#endif
gasnete_eop_t *pop = (gasnete_eop_t *) context;
#if defined(GASNET_DEBUG) || defined(GASNETI_STATS_OR_TRACE)
gasnet_node_t node = pop->node;
gasneti_tick_t starttime = GASNETI_TICKS_NOW_IFENABLED(C);
#endif
const firehose_request_t *fhreqs[2];
int numreqs = 1;
gasneti_mutex_assertlocked(&gasnetc_lock_gm);
gasneti_assert(pop != NULL);
gasneti_assert(node != gasneti_mynode && node < gasneti_nodes);
if_pf (status != GM_SUCCESS)
gasnetc_callback_error(status, NULL);
gasnetc_token_lo_release();
GASNETI_TRACE_PRINTF(C,
("Firehose decrement remote refcount for (%p,%d) on node %d (op=%p,%p,%d)\n",
(void *) pop->dest, pop->len, (unsigned) node, (void *) pop,
(void *)pop->req_remote.addr, (int)pop->req_remote.len));
fhreqs[0] = &(pop->req_remote);
/* Puts use an ambuffer, while bulk puts send from a pinned location */
if (OPMISC(pop) == OPMISC_AMBUF) {
gasnetc_bufdesc_t *bufd;
bufd = (gasnetc_bufdesc_t *) GASNETC_BUFDESC_PTR(pop->src);
GASNETC_ASSERT_BUFDESC_PTR(bufd, pop->src);
gasnetc_callback_ambuffer(p, (void *) bufd, status);
}
else {
fhreqs[1] = pop->req_local;
numreqs++;
#ifdef GASNET_SEGMENT_FAST
/* Only release locally for "fast" segment */
firehose_release(fhreqs+1, 1);
#endif
}
#ifndef GASNET_SEGMENT_FAST
GASNETE_GM_SET_IN_UNKNOWN();
firehose_release(fhreqs, numreqs);
GASNETE_GM_UNSET_IN_UNKNOWN();
#endif
GASNETE_FIREHOSE_TRACE_PUTGET(pop, PUT);
/* If this was associated to an iop, increment put completed count */
if (pop->iop != NULL) {
gasneti_weakatomic_increment(&(pop->iop->completed_put_cnt),0/*Rel?*/);
gasneti_free(pop); /* free a "dummy" eop */
} else {
gasnete_op_markdone((gasnete_op_t *)pop, 0);
}
GASNETI_TRACE_EVENT_TIME(C, FIREHOSE_MOVE_LOCAL,
GASNETI_TICKS_NOW_IFENABLED(C)-starttime);
return;
}
