void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
mpi_buf_t buf = {0, 0, NULL};
SNetRefSerialise(ref, &buf, &PackInt, &PackByte);
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &buf);
MPISend(buf.data, buf.offset, (uintptr_t) dest, snet_ref_set);
SNetMemFree(buf.data);
}
void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
// Only called from imanager, so no safety issue to use static
static mpi_buf_t buf = {0, 0, NULL};
buf.offset = 0;
SNetRefSerialise(ref, &buf);
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &buf);
MPISendBuf(&buf, (uintptr_t) dest, snet_ref_set);
}
snet_msg_t SNetDistribRecvMsg(void)
{
int count;
snet_msg_t result;
MPI_Status status;
static mpi_buf_t recvBuf = {0, 0, NULL};
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, MPI_PACKED, &count);
MPI_Pack_size(count, MPI_PACKED, MPI_COMM_WORLD, &recvBuf.offset);
if (recvBuf.offset > recvBuf.size) {
recvBuf.data = SNetMemResize(recvBuf.data, recvBuf.offset);
recvBuf.size = recvBuf.offset;
}
MPI_Recv(recvBuf.data, count, MPI_PACKED, status.MPI_SOURCE, status.MPI_TAG,
MPI_COMM_WORLD, &status);
recvBuf.offset = 0;
result.type = status.MPI_TAG;
switch (status.MPI_TAG) {
case snet_rec:
result.rec = SNetRecDeserialise(&recvBuf);
case snet_block:
case snet_unblock:
result.dest = SNetDestDeserialise(&recvBuf);
result.dest.node = status.MPI_SOURCE;
break;
case snet_ref_set:
result.ref = SNetRefDeserialise(&recvBuf);
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&recvBuf);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise(&recvBuf);
result.data = status.MPI_SOURCE;
break;
case snet_ref_update:
result.ref = SNetRefDeserialise(&recvBuf);
SNetDistribUnpack(&recvBuf, &result.val, sizeof(result.val));
break;
case snet_update:
break;
case snet_stop:
break;
default:
SNetUtilDebugFatal("[%s]: Unexpected MPI TAG %d\n", __func__, result.type);
break;
}
return result;
}
snet_msg_t SNetDistribRecvMsg(void)
{
int count;
snet_msg_t result;
MPI_Status status;
static mpi_buf_t recvBuf = {0, 0, NULL};
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, MPI_PACKED, &count);
MPI_Pack_size(count, MPI_PACKED, MPI_COMM_WORLD, &recvBuf.offset);
if ((unsigned) recvBuf.offset > recvBuf.size) {
SNetMemFree(recvBuf.data);
recvBuf.data = SNetMemAlloc(recvBuf.offset);
recvBuf.size = recvBuf.offset;
}
MPI_Recv(recvBuf.data, count, MPI_PACKED, MPI_ANY_SOURCE, status.MPI_TAG,
MPI_COMM_WORLD, &status);
recvBuf.offset = 0;
result.type = status.MPI_TAG;
switch (result.type) {
case snet_rec:
result.rec = SNetRecDeserialise(&recvBuf, &UnpackInt, &UnpackRef);
case snet_block:
case snet_unblock:
result.dest.node = status.MPI_SOURCE;
UnpackDest(&recvBuf, &result.dest);
break;
case snet_ref_set:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&recvBuf);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
result.data = status.MPI_SOURCE;
break;
case snet_ref_update:
result.ref = SNetRefDeserialise(&recvBuf, &UnpackInt, &UnpackByte);
UnpackInt(&recvBuf, 1, &result.val);
break;
default:
break;
}
return result;
}
void SNetDistribSendData(snet_ref_t *ref, void *data, void *dest)
{
int node;
lut_addr_t addr;
snet_comm_type_t type = snet_ref_set;
if (remap) {
addr.node = *(int*) dest;
} else {
addr = *(lut_addr_t*) dest;
SNetMemFree(dest);
}
node = addr.node;
start_write_node(node);
cpy_mem_to_mpb(node, &type, sizeof(snet_comm_type_t));
SNetRefSerialise(ref, (void*) node, &PackInt, &PackByte);
if (!remap) cpy_mem_to_mpb(node, &addr, sizeof(lut_addr_t));
SNetInterfaceGet(SNetRefInterface(ref))->packfun(data, &addr);
stop_write_node(node);
}
snet_msg_t SNetDistribRecvMsg(void)
{
lut_addr_t addr;
snet_msg_t result;
static sigset_t sigmask;
static bool handling = true, set = false;
if (!set) {
set = true;
write_pid();
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGUSR1);
sigaddset(&sigmask, SIGUSR2);
}
start:
if (!handling) {
sigwait(&sigmask, &result.val);
if (result.val == SIGUSR2) {
result.type = snet_update;
return result;
}
}
lock(node_location);
flush();
if (START(node_location) == END(node_location)) {
handling = false;
HANDLING(node_location) = 0;
FOOL_WRITE_COMBINE;
unlock(node_location);
goto start;
} else if (!handling) {
handling = true;
HANDLING(node_location) = 1;
FOOL_WRITE_COMBINE;
}
cpy_mpb_to_mem(node_location, &result.type, sizeof(snet_comm_type_t));
switch (result.type) {
case snet_rec:
result.rec = SNetRecDeserialise((void*) node_location, &UnpackInt, &UnpackRef);
case snet_block:
case snet_unblock:
cpy_mpb_to_mem(node_location, &result.dest, sizeof(snet_dest_t));
break;
case snet_ref_set:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
if (!remap) cpy_mpb_to_mem(node_location, &addr, sizeof(lut_addr_t));
result.data = (uintptr_t) SNetInterfaceGet(SNetRefInterface(result.ref))->unpackfun(&addr);
break;
case snet_ref_fetch:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
if (remap) {
cpy_mpb_to_mem(node_location, &result.val, sizeof(int));
unlock(node_location);
SNetDistribSendData(result.ref, SNetRefGetData(result.ref), &result.val);
SNetMemFree(result.ref);
goto start;
} else {
result.data = (uintptr_t) SNetMemAlloc(sizeof(lut_addr_t));
cpy_mpb_to_mem(node_location, (void*) result.data, sizeof(lut_addr_t));
}
break;
case snet_ref_update:
result.ref = SNetRefDeserialise((void*) node_location, &UnpackInt, &UnpackByte);
cpy_mpb_to_mem(node_location, &result.val, sizeof(int));
break;
default:
break;
}
unlock(node_location);
return result;
}
/* This function prints records to stdout */
static void printRec(snet_record_t *rec, handle_t *hnd)
{
snet_ref_t *field;
int name, val;
char *label = NULL;
char *interface = NULL;
snet_record_mode_t mode;
/* Change this to redirect the output! */
if (rec != NULL) {
fprintf(hnd->file, "<?xml version=\"1.0\" ?>\n");
switch( SNetRecGetDescriptor( rec)) {
case REC_data:
mode = SNetRecGetDataMode(rec);
if (mode == MODE_textual) {
fprintf(hnd->file, "<record xmlns=\"snet-home.org\" type=\"data\" mode=\"textual\" >\n");
} else {
fprintf(hnd->file, "<record xmlns=\"snet-home.org\" type=\"data\" mode=\"binary\" >\n");
}
/* Fields */
RECORD_FOR_EACH_FIELD(rec, name, field) {
int id = SNetRecGetInterfaceId(rec);
if((label = SNetInIdToLabel(hnd->labels, name)) != NULL){
if((interface = SNetInIdToInterface(hnd->interfaces, id)) != NULL) {
fprintf(hnd->file, "<field label=\"%s\" interface=\"%s\">", label,
interface);
if(mode == MODE_textual) {
SNetInterfaceGet(id)->serialisefun(hnd->file,
SNetRefGetData(field));
} else {
SNetInterfaceGet(id)->encodefun(hnd->file,
SNetRefGetData(field));
}
fprintf(hnd->file, "</field>\n");
SNetMemFree(interface);
}
SNetMemFree(label);
} else{
SNetUtilDebugFatal("Unknown field %d at output!", name);
}
}
/* Tags */
RECORD_FOR_EACH_TAG(rec, name, val) {
if ((label = SNetInIdToLabel(hnd->labels, name)) != NULL) {
fprintf(hnd->file, "<tag label=\"%s\">%d</tag>\n", label, val);
} else{
SNetUtilDebugFatal("Unknown tag %d at output!", name);
}
SNetMemFree(label);
}
/* BTags */
RECORD_FOR_EACH_BTAG(rec, name, val) {
if ((label = SNetInIdToLabel(hnd->labels, name)) != NULL){
fprintf(hnd->file, "<btag label=\"%s\">%d</btag>\n", label, val);
} else{
SNetUtilDebugFatal("Unknown binding tag %d at output!", name);
}
SNetMemFree(label);
}
fprintf(hnd->file, "</record>\n");
break;
case REC_sync:
SNetUtilDebugFatal("REC_sync in output! This should not happen.");
break;
case REC_collect:
SNetUtilDebugFatal("REC_collect in output! This should not happen.");
break;
case REC_sort_end:
SNetUtilDebugFatal("REC_sort_end in output! This should not happen.");
break;
case REC_trigger_initialiser:
SNetUtilDebugFatal("REC_trigger_initializer in output! This should not happen.");
break;
case REC_terminate:
fprintf(hnd->file, "<record type=\"terminate\" />");
break;
default:
break;
}
