void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
mpi_buf_t buf = {0, 0, NULL};
SNetRefSerialise(ref, &buf, &PackInt, &PackByte);
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &buf);
MPISend(buf.data, buf.offset, (uintptr_t) dest, snet_ref_set);
SNetMemFree(buf.data);
}
void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
// Only called from imanager, so no safety issue to use static
static mpi_buf_t buf = {0, 0, NULL};
buf.offset = 0;
SNetRefSerialise(ref, &buf);
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &buf);
MPISendBuf(&buf, (uintptr_t) dest, snet_ref_set);
}
snet_msg_t SNetDistribRecvMsg(void)
{
int count;
snet_msg_t result;
MPI_Status status;
static mpi_buf_t recvBuf = {0, 0, NULL};
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, MPI_PACKED, &count);
MPI_Pack_size(count, MPI_PACKED, MPI_COMM_WORLD, &recvBuf.offset);
if (recvBuf.offset > recvBuf.size) {
recvBuf.data = SNetMemResize(recvBuf.data, recvBuf.offset);
recvBuf.size = recvBuf.offset;
}
MPI_Recv(recvBuf.data, count, MPI_PACKED, status.MPI_SOURCE, status.MPI_TAG,
MPI_COMM_WORLD, &status);
recvBuf.offset = 0;
result.type = status.MPI_TAG;
switch (status.MPI_TAG) {
case snet_rec:
result.rec = SNetRecDeserialise(&recvBuf);
case snet_block:
case snet_unblock:
result.dest = SNetDestDeserialise(&recvBuf);
result.dest.node = status.MPI_SOURCE;
break;
case snet_ref_set:
result.ref = SNetRefDeserialise(&recvBuf);
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&recvBuf);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise(&recvBuf);
result.data = status.MPI_SOURCE;
break;
case snet_ref_update:
result.ref = SNetRefDeserialise(&recvBuf);
SNetDistribUnpack(&recvBuf, &result.val, sizeof(result.val));
break;
case snet_update:
break;
case snet_stop:
break;
default:
SNetUtilDebugFatal("[%s]: Unexpected MPI TAG %d\n", __func__, result.type);
break;
}
return result;
}
snet_msg_t SNetDistribRecvMsg(void)
{
int count;
snet_msg_t result;
MPI_Status status;
static mpi_buf_t recvBuf = {0, 0, NULL};
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, MPI_PACKED, &count);
MPI_Pack_size(count, MPI_PACKED, MPI_COMM_WORLD, &recvBuf.offset);
if ((unsigned) recvBuf.offset > recvBuf.size) {
SNetMemFree(recvBuf.data);
recvBuf.data = SNetMemAlloc(recvBuf.offset);
recvBuf.size = recvBuf.offset;
}
MPI_Recv(recvBuf.data, count, MPI_PACKED, MPI_ANY_SOURCE, status.MPI_TAG,
MPI_COMM_WORLD, &status);
recvBuf.offset = 0;
result.type = status.MPI_TAG;
switch (result.type) {
case snet_rec:
result.rec = SNetRecDeserialise(&recvBuf, &UnpackInt, &UnpackRef);
case snet_block:
case snet_unblock:
result.dest.node = status.MPI_SOURCE;
UnpackDest(&recvBuf, &result.dest);
break;
case snet_ref_set:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&recvBuf);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
result.data = status.MPI_SOURCE;
break;
case snet_ref_update:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
UnpackInt(&recvBuf, 1, &result.val);
break;
default:
break;
}
return result;
}
void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
int node;
lut_addr_t addr;
snet_comm_type_t type = snet_ref_set;
if (remap) {
addr.node = *(int*) dest;
} else {
addr = *(lut_addr_t*) dest;
SNetMemFree(dest);
}
node = addr.node;
start_write_node(node);
cpy_mem_to_mpb(node, &type, sizeof(snet_comm_type_t));
SNetRefSerialise(ref, (void*) node, &PackInt, &PackByte);
if (!remap) cpy_mem_to_mpb(node, &addr, sizeof(lut_addr_t));
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &addr);
stop_write_node(node);
}
snet_msg_t SNetDistribRecvMsg(void)
{
lut_addr_t addr;
snet_msg_t result;
static sigset_t sigmask;
static bool handling = true, set = false;
if (!set) {
set = true;
write_pid();
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGUSR1);
sigaddset(&sigmask, SIGUSR2);
}
start:
if (!handling) {
sigwait(&sigmask, &result.val);
if (result.val == SIGUSR2) {
result.type = snet_update;
return result;
}
}
lock(node_location);
flush();
if (START(node_location) == END(node_location)) {
handling = false;
HANDLING(node_location) = 0;
FOOL_WRITE_COMBINE;
unlock(node_location);
goto start;
} else if (!handling) {
handling = true;
HANDLING(node_location) = 1;
FOOL_WRITE_COMBINE;
}
cpy_mpb_to_mem(node_location, &result.type, sizeof(snet_comm_type_t));
switch (result.type) {
case snet_rec:
result.rec = SNetRecDeserialise((void*) node_location, &UnpackInt, &UnpackRef);
case snet_block:
case snet_unblock:
cpy_mpb_to_mem(node_location, &result.dest, sizeof(snet_dest_t));
break;
case snet_ref_set:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
if (!remap) cpy_mpb_to_mem(node_location, &addr, sizeof(lut_addr_t));
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&addr);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
if (remap) {
cpy_mpb_to_mem(node_location, &result.val, sizeof(int));
unlock(node_location);
SNetDistribSendData(result.ref, SNetRefGetData(result.ref), &result.val);
SNetMemFree(result.ref);
goto start;
} else {
result.data = (uintptr_t) SNetMemAlloc(sizeof(lut_addr_t));
cpy_mpb_to_mem(node_location, (void*) result.data, sizeof(lut_addr_t));
}
break;
case snet_ref_update:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
cpy_mpb_to_mem(node_location, &result.val, sizeof(int));
break;
default:
break;
}
unlock(node_location);
return result;
}
