int CBTF_MRNet_LW_connect (const int con_rank)
{
pthread_mutex_lock(&mrnet_connected_mutex);
if (mrnet_connected) {
pthread_mutex_unlock(&mrnet_connected_mutex);
return 1;
}
const char* connfile = getenv("CBTF_MRNETBE_CONNECTIONS");
if (connfile == NULL) {
const char* connections_dir = getenv("PWD");
char buf[4096];
sprintf(buf,"%s%s",connections_dir,"/attachBE_connections");
connfile = strdup(buf);
}
char parHostname[64], myHostname[64], parPort[10], parRank[10], myRank[10];
/* sequential jobs pass con_rank of -1. In that case we need to
* use a rank of 0 and mRank of 10000 for the connection.
*/
Rank mRank;
int rank_to_use = con_rank;
if (con_rank < 0 ) {
mRank = 10000;
rank_to_use = 0;
} else {
mRank = 10000 + con_rank;
}
sprintf(myRank, "%d", mRank);
if( CBTF_MRNet_getParentInfo(connfile, rank_to_use, parHostname, parPort, parRank) != 0 ) {
fprintf(stderr, "CBTF_MRNet_LW_connect: Failed to parse connections file %s\n",connfile);
fprintf(stderr, "CBTF_MRNet_LW_connect: Failed for myRank %s, mrank %d, con_rank %d\n",myRank,mRank,rank_to_use);
abort();
}
while( gethostname(myHostname, 64) == -1 ) {}
myHostname[63] = '\0';
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr, "CBTF_MRNet_LW_connect: myRank %s, mRank %d, host %s\n",myRank,mRank,myHostname);
}
#endif
int BE_argc = 6;
char* BE_argv[6];
char be_name[24];
sprintf(be_name,"%s%s","cbtfBE",myRank);
BE_argv[0] = be_name;
BE_argv[1] = parHostname;
BE_argv[2] = parPort;
BE_argv[3] = parRank;
BE_argv[4] = myHostname;
BE_argv[5] = myRank;
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr,"CBTF_MRNet_LW_connect: argv 0=%s, 1=%s, 2=%lu, 3=%lu, 4=%s, 5=%lu\n",
BE_argv[0], BE_argv[1], strtoul( BE_argv[2], NULL, 10 ),
strtoul( BE_argv[3], NULL, 10 ), BE_argv[4], strtoul( BE_argv[5], NULL, 10 ));
fprintf( stderr, "CBTF_MRNet_LW_connect: Backend %s[%s] connecting to %s:%s[%s]\n",
myHostname,myRank,parHostname,parPort,myRank);
}
#endif
Packet_t * p;
p = (Packet_t *)malloc(sizeof(Packet_t));
Assert(p);
CBTF_MRNet_netPtr = Network_CreateNetworkBE(BE_argc, BE_argv);
Assert(CBTF_MRNet_netPtr);
int tag;
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr, "CBTF_MRNet_LW_connect: TRYING TO ESTABLISH CBTF_MRNet_stream\n");
}
#endif
Stream_t* CBTF_MRNet_stream;
if (Network_recv(CBTF_MRNet_netPtr, &tag, p, &CBTF_MRNet_stream) != 1) {
fprintf(stderr, "CBTF_MRNet_LW_connect: BE receive failure\n");
abort();
}
stream_id = CBTF_MRNet_stream->id;
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr,
"CBTF_MRNet_LW_connect: got tag %d, stream id %d, sync_filter_id %d, us_filter_id %d, ds_filter_id %d\n",
tag, CBTF_MRNet_stream->id,CBTF_MRNet_stream->sync_filter_id,
CBTF_MRNet_stream->us_filter_id, CBTF_MRNet_stream->ds_filter_id);
}
#endif
/* wait for FE to notify netowrk is ready
* Use the SpecifyFilter tag for this since once all commnodes
* have loaded a filter, this message is passed down to the BE.
* This holds the collectors from continuing until any needed
* filter is loaded above.
*/
if (CBTF_MRNet_netPtr) {
/* get our stream */
Stream_t* stream = Network_get_Stream(CBTF_MRNet_netPtr,stream_id);
int readytag = 0;
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr, "BE: waiting for specify filter tag from FE\n");
}
#endif
/* use KRELL_INSTITUTE_CBTF_IMPL_SPECIFY_FILTER for readytag 105
* include/from KrellInstitute/CBTF/Impl/MessageTags.h
*/
do {
if( CBTF_MRNet_netPtr && Network_recv(CBTF_MRNet_netPtr, &readytag, p, &stream) != 1 ) {
fprintf(stderr, "BE: receive failure\n");
break;
}
} while ( readytag != KRELL_INSTITUTE_CBTF_IMPL_NETWORK_READY /*107*/ );
#ifndef NDEBUG
if (IsMRNetDebugEnabled) {
fprintf(stderr, "BE: GOT TAG for filter tag %d from FE\n",readytag);
}
#endif
}
#if defined(ENABLE_CBTF_MRNET_PLAYBACK)
playback_configure(Network_get_LocalRank(CBTF_MRNet_netPtr));
#endif
mrnet_connected = 1;
pthread_cond_broadcast(&mrnet_connected_cond);
pthread_mutex_unlock(&mrnet_connected_mutex);
if (p != NULL) {
free(p);
}
return 1;
}
int main(int argc, char **argv)
{
Stream_t * stream;
Packet_t* pkt = (Packet_t*)malloc(sizeof(Packet_t));
Network_t * net;
int tag=0;
uint32_t recv_array_len=0;
void * recv_array=NULL;
int success=1;
Rank my_rank;
assert(pkt);
net = Network_CreateNetworkBE( argc, argv );
my_rank = Network_get_LocalRank(net);
do{
if ( Network_recv(net, &tag, pkt, &stream) != 1){
fprintf(stderr, "BE(%d): net::recv() failure\n", my_rank);
}
recv_array=NULL;
switch(tag){
case PROT_CHAR:
//fprintf(stdout, "BE(%d): Processing PROT_CHAR_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%ac", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%ac) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%ac", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%ac) failure\n", my_rank);
success=0;
}
break;
case PROT_UCHAR:
//fprintf(stdout, "BE(%d): Processing PROT_UCHAR_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%auc", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%auc) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%auc", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%auc) failure\n", my_rank);
success=0;
}
break;
case PROT_INT:
//fprintf(stdout, "BE(%d): Processing PROT_INT_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%ad", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%ad) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%ad", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%ad) failure\n", my_rank);
success=0;
}
break;
case PROT_UINT:
//fprintf(stdout, "BE(%d): Processing PROT_UINT_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%aud", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%aud) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%aud", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%aud) failure\n", my_rank);
success=0;
}
break;
case PROT_SHORT:
//fprintf(stdout, "BE(%d): Processing PROT_SHORT_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%ahd", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%ahd) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%ahd", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%ahd) failure\n", my_rank);
success=0;
}
break;
case PROT_USHORT:
//fprintf(stdout, "BE(%d): Processing PROT_USHORT_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%auhd", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%auhd) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%auhd", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%auhd) failure\n", my_rank);
success=0;
}
break;
case PROT_LONG:
//fprintf(stdout, "BE(%d): Processing PROT_LONG_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%ald", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%ald) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%ald", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%ald) failure\n", my_rank);
success=0;
}
break;
case PROT_ULONG:
//fprintf(stdout, "BE(%d): Processing PROT_ULONG_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%auld", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%auld) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%auld", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%auld) failure\n", my_rank);
success=0;
}
break;
case PROT_FLOAT:
//fprintf(stdout, "BE(%d): Processing PROT_FLOAT_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%af", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%af) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%af", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%af) failure\n", my_rank);
success=0;
}
break;
case PROT_DOUBLE:
//fprintf(stdout, "BE(%d): Processing PROT_DOUBLE_ARRAY ...\n", my_rank);
if( Packet_unpack(pkt, "%alf", &recv_array, &recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::unpack(%%alf) failure\n", my_rank);
success=0;
}
if( Stream_send(stream,tag, "%alf", recv_array, recv_array_len) == -1 ){
fprintf(stderr, "BE(%d): stream::send(%%alf) failure\n", my_rank);
success=0;
}
break;
case PROT_EXIT:
//fprintf(stdout, "BE(%d): Processing PROT_EXIT ...\n", my_rank);
break;
default:
fprintf(stderr, "BE(%d): Unknown Protocol: %d\n", my_rank, tag);
exit(-1);
}
//fflush(stdout);
if( tag != PROT_EXIT ) {
if( Stream_flush(stream) == -1 ) {
fprintf(stderr, "BE(%d): stream::flush() failure\n", my_rank);
return -1;
}
}
} while( tag != PROT_EXIT );
if( pkt != NULL )
free(pkt);
// wait for final teardown packet from FE; this will cause
// us to exit
Network_waitfor_ShutDown(net);
if( net != NULL )
delete_Network_t(net);
return 0;
}
void* BackendThreadMain( void* arg )
{
info_t* rinfo = (info_t*)arg;
int32_t recv_int=0;
int tag;
Packet_t* p;
Stream_t* stream;
Network_t * net = NULL;
const char* fmt_str = "%d";
char parHostname[64], parPort[10], parRank[10], myRank[10];
Rank mRank = rinfo->mrnet_rank;
sprintf(myRank, "%d", mRank);
if( getParentInfo(connfile, rinfo->mpi_rank, parHostname, parPort, parRank) != 0 ) {
fprintf(stderr, "Failed to parse connections file\n");
return NULL;
}
assert( rinfo->argc == 6 );
fprintf( stdout, "BE[%d] on %s connecting to %s:%s[%s]\n",
mRank, rinfo->argv[4], parHostname, parPort, parRank );
rinfo->argv[1] = parHostname;
rinfo->argv[2] = parPort;
rinfo->argv[3] = parRank;
rinfo->argv[5] = myRank;
net = Network_CreateNetworkBE( rinfo->argc, rinfo->argv );
p = (Packet_t*) calloc( (size_t)1, sizeof(Packet_t) );
do {
if( Network_recv(net, &tag, p, &stream) != 1 ) {
printf("net->recv() failure\n");
tag = PROT_EXIT;
}
switch( tag ) {
case PROT_INT:
if( Packet_unpack( p, fmt_str, &recv_int) == -1 ) {
printf("Packet_unpack() failure\n");
return NULL;
}
fprintf(stdout, "BE[%d]: received int = %d\n", mRank, recv_int);
if( (Stream_send(stream, PROT_INT, fmt_str, recv_int) == -1) ||
(Stream_flush(stream) == -1 ) ) {
printf("Stream_send() failure\n");
return NULL;
}
break;
case PROT_EXIT:
fprintf(stdout, "BE[%d]: received PROT_EXIT\n", mRank);
break;
default:
fprintf(stdout, "BE[%d]: Unknown Protocol %d\n", mRank, tag);
tag = PROT_EXIT;
break;
}
fflush(stdout);
fflush(stderr);
} while ( tag != PROT_EXIT );
// FE delete of the network will cause us to exit, wait for it
Network_waitfor_ShutDown(net);
delete_Network_t(net);
return NULL;
}
