int BackEndNode_proc_deleteStream( BackEndNode_t* be, Packet_t* ipacket )
{
unsigned int stream_id;
Stream_t * strm;
mrn_dbg_func_begin();
if (Packet_unpack(ipacket, "%ud", &stream_id) == -1) {
mrn_dbg(1, mrn_printf(FLF, stderr, "Packet_unpack() failed\n"));
return -1;
}
strm = Network_get_Stream(be->network, stream_id);
if(Network_is_UserStreamId(stream_id)) {
if (strm != NULL) {
// close user streams
strm->_was_closed = 1;
} else {
mrn_dbg(1, mrn_printf(FLF, stderr, "stream %u lookup failed\n", stream_id));
return -1;
}
} else {
if( strm != NULL ) {
// kill internal streams
delete_Stream_t( strm );
}
}
mrn_dbg_func_end();
return 0;
}
SerialGraph_t* new_SerialGraph_t(char* ibyte_array)
{
SerialGraph_t* sg;
mrn_dbg_func_begin();
sg = (SerialGraph_t*) calloc( (size_t)1, sizeof(SerialGraph_t) );
assert(sg != NULL);
if( ibyte_array != NULL ) {
sg->byte_array = strdup(ibyte_array);
sg->arr_len = strlen(ibyte_array) + 1;
mrn_dbg(5, mrn_printf(FLF, stderr,
"Creating new serial graph with byte_array='%s'\n",
sg->byte_array));
} else {
sg->arr_len = 1024; /* pre-allocation */
sg->byte_array = (char*) malloc((size_t)sg->arr_len);
assert(sg->byte_array);
sg->byte_array[0] = '\0';
mrn_dbg(5, mrn_printf(FLF, stderr,
"Creating new serial graph with empty byte_array\n"));
}
mrn_dbg_func_end();
return sg;
}
void SerialGraph_add_SubTreeRoot(SerialGraph_t* sg, char* ihostname,
Port iport, Rank irank)
{
char hoststr[256];
size_t len;
mrn_dbg_func_begin();
len = (size_t) sprintf(hoststr, "[%s:%05hu:%u:1", ihostname, iport, irank);
mrn_dbg(5, mrn_printf(FLF, stderr, "adding sub tree root: %s\n", hoststr));
len += strlen(sg->byte_array) + 1;
if( len > sg->arr_len ) {
len *= 2;
sg->byte_array = (char*) realloc(sg->byte_array, len);
if( sg->byte_array == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "realloc(%" PRIszt ") failed\n", len));
exit(0);
}
sg->arr_len = len;
}
strcat(sg->byte_array, hoststr);
sg->num_nodes++;
mrn_dbg_func_end();
}
SerialGraph_t* SerialGraph_get_MySubTree(SerialGraph_t* sg, char* ihostname,
Port iport, Rank irank)
{
char hoststr[256];
size_t cur, end;
int num_leftbrackets=1, num_rightbrackets=0;
char *byte_array, *new_byte_array;
SerialGraph_t* retval;
mrn_dbg_func_begin();
sprintf(hoststr, "[%s:%05hu:%u:", ihostname, iport, irank);
mrn_dbg(5, mrn_printf(FLF, stderr, "looking for SubTreeRoot: '%s'\n", hoststr));
byte_array = sg->byte_array;
byte_array = strstr(byte_array, hoststr);
if (byte_array == NULL) {
mrn_dbg(3, mrn_printf(FLF, stderr,
"No SubTreeRoot: '%s' found in byte_array: '%s'\n",
hoststr, sg->byte_array));
return NULL;
}
//now find matching ']'
cur = 0;
end = 1;
while( num_leftbrackets != num_rightbrackets ) {
cur++;
end++;
if(byte_array[cur] == '[')
num_leftbrackets++;
else if (byte_array[cur] == ']')
num_rightbrackets++;
}
new_byte_array = (char*) malloc((size_t)end+1);
if( new_byte_array == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "malloc(%" PRIszt ") failed\n", end+1));
exit(0);
}
strncpy(new_byte_array, byte_array, end);
new_byte_array[end] = '\0';
retval = new_SerialGraph_t(new_byte_array);
// free malloc'd stuff
free(new_byte_array);
mrn_dbg(5, mrn_printf(FLF, stderr, "returned sg byte array '%s'\n",
retval->byte_array));
mrn_dbg_func_end();
return retval;
}
unsigned int ParsedGraph::preorder_traversal( Node * node )
{
unsigned int nodes_visited=0;
if( node->visited() == true ){
mrn_dbg(1, mrn_printf(FLF, stderr, "%s:%u: Node is own ancestor",
node->_hostname.c_str(), node->_local_rank ) );
error( ERR_TOPOLOGY_CYCLE, UnknownRank, "%s:%u: Node is own ancestor",
node->_hostname.c_str(), node->_local_rank );
_cycle_free=false;
return 0;
}
else{
node->visit();
nodes_visited++;
if(node->_children.size() == 0){
return nodes_visited;
}
}
for(unsigned int i=0; i<node->_children.size(); i++){
nodes_visited+=preorder_traversal(node->_children[i]);
}
return nodes_visited;
}
// BE and IN constructor
RSHNetwork::RSHNetwork( const char* phostname, Port pport, Rank prank,
const char* myhostname, Rank myrank, bool isInternal )
{
if( isInternal ) {
// initialize as a Network IN
Network::init_InternalNode( phostname,
pport,
prank,
myhostname,
myrank );
RSHInternalNode* in = dynamic_cast<RSHInternalNode*>( get_LocalInternalNode() );
if( in == NULL )
mrn_dbg( 1, mrn_printf(FLF, stderr, "internal-node dynamic_cast failed\n") );
else
in->request_LaunchInfo();
}
else {
// initialize as a Network BE
Network::init_BackEnd( phostname,
pport,
prank,
myhostname,
myrank );
}
}
InternalNode*
RSHNetwork::CreateInternalNode( Network* inetwork,
std::string imy_hostname,
Rank imy_rank,
std::string iphostname,
Port ipport,
Rank iprank,
int idataSocket,
Port idataPort )
{
XPlat_Socket listeningSocket = idataSocket;
Port listeningPort = idataPort;
// we build and bind our listening socket now
// because other platforms need to have this information
// in their initialization code, and it doesn't hurt us
// to bind it early
int bRet = bindPort( &listeningSocket, &listeningPort, true /*non-blocking*/ );
if( bRet == -1 ) {
mrn_dbg( 1, mrn_printf(FLF, stderr,
"Failure: unable to instantiate network\n") );
return NULL;
}
assert( listeningPort != UnknownPort );
return new RSHInternalNode( inetwork,
imy_hostname,
imy_rank,
iphostname,
ipport,
iprank,
listeningSocket,
listeningPort );
}
int SerialGraph_is_RootBackEnd(SerialGraph_t* sg)
{
//format is "host:port:rank:[0|1]: where 0 means back-end and 1 means internal node
size_t idx;
unsigned int num_colons;
char typ;
for( idx=0, num_colons=0; num_colons < 3; idx++ ) {
if( ':' == sg->byte_array[idx] ) {
num_colons++;
}
}
typ = sg->byte_array[idx];
if( typ == '0' ) {
return 1;
}
else if ( typ == '1' ) {
return 0;
}
else {
mrn_dbg(1, mrn_printf(FLF, stderr,
"unexpected character '%c' in node type field\n", typ));
assert(false);
}
// Should never get here
return -1;
}
void SerialGraph_set_ToFirstChild(SerialGraph_t* sg)
{
char* buf;
const char* delim;
size_t val;
buf = sg->byte_array;
delim = "[";
val = strcspn(buf+1, delim);
sg->buf_idx = val + 1;
mrn_dbg(5, mrn_printf(FLF, stderr, "set_ToFirstChild: buf_idx=%" PRIszt "\n",
sg->buf_idx));
}
bool
RSHNetwork::Instantiate( ParsedGraph* _parsed_graph,
const char* mrn_commnode_path,
const char* backend_exe,
const char** backend_args,
unsigned int backend_argc,
const std::map< std::string, std::string >* /* iattrs */ )
{
bool have_backends = ( strlen(backend_exe) != 0 );
std::string sg = _parsed_graph->get_SerializedGraphString( have_backends );
NetworkTopology* nt = get_NetworkTopology();
if( nt != NULL ) {
nt->reset( sg, false );
NetworkTopology::Node* localnode = nt->find_Node( get_LocalRank() );
localnode->set_Port( get_LocalPort() );
}
RSHFrontEndNode* fe = dynamic_cast<RSHFrontEndNode*>( get_LocalFrontEndNode() );
if( fe == NULL ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "front-end dynamic_cast failed\n") );
return false;
}
mrn_dbg( 5, mrn_printf(FLF, stderr, "Instantiating network\n") );
PacketPtr packet( new MRN::Packet(CTL_STRM_ID, PROT_LAUNCH_SUBTREE, "%s %s %as",
mrn_commnode_path,
backend_exe,
backend_args, backend_argc) );
if( fe->proc_LaunchSubTree(packet) == -1 ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "failed to spawn subtree\n") );
error( ERR_INTERNAL, get_LocalRank(), "failed to spawn subtree");
return false;
}
return true;
}
bool ParsedGraph::validate()
{
unsigned int visited_nodes = preorder_traversal( _root );
if( visited_nodes != _nodes.size() ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "Failure: "
"visited nodes(%d) != total nodes(%d)!\n",
visited_nodes, _nodes.size() ));
error( ERR_TOPOLOGY_NOTCONNECTED, UnknownRank, "parsed graph not connected" );
_fully_connected = false;
}
fflush(stdout);
return ( _cycle_free && _fully_connected );
}
int BackEndNode_proc_DataFromParent( BackEndNode_t* be, Packet_t* ipacket )
{
Stream_t* stream;
Packet_t* opacket;
vector_t * opackets = new_empty_vector_t();
vector_t * opackets_reverse = new_empty_vector_t();
unsigned int i;
mrn_dbg_func_begin();
stream = Network_get_Stream(be->network, ipacket->stream_id);
if (stream == NULL) {
mrn_dbg(1, mrn_printf(FLF, stderr, "stream %u lookup failed\n",
ipacket->stream_id));
return -1;
}
if (Stream_push_Packet(stream, ipacket, opackets, opackets_reverse, false) == -1) {
return -1;
}
for (i = 0; i < opackets->size; i++) {
opacket = opackets->vec[i];
Packet_set_DestroyData( opacket, true );
mrn_dbg(5, mrn_printf(FLF, stderr, "adding Packet on stream[%u]\n",
stream->id));
Stream_add_IncomingPacket(stream, opacket);
}
delete_vector_t( opackets );
delete_vector_t( opackets_reverse );
mrn_dbg_func_end();
return 0;
}
SerialGraph_t* SerialGraph_get_NextChild(SerialGraph_t* sg)
{
size_t end, cur;
int num_leftbrackets=1, num_rightbrackets=0;
char *buf, *buf2, *buf3;
SerialGraph_t* retval;
mrn_dbg_func_begin();
cur = sg->buf_idx;
if( cur > sg->arr_len )
return NULL;
buf = sg->byte_array;
buf2 = buf + cur;
if( NULL == strchr(buf2, '[') ) {
return NULL;
}
end = 1;
while( num_leftbrackets != num_rightbrackets ) {
cur++;
end++;
if( buf[cur] == '[' )
num_leftbrackets++;
else if( buf[cur] == ']' )
num_rightbrackets++;
}
sg->buf_idx = cur + 1;
// get substr from start pos to end
buf3 = (char*) malloc(end+1);
assert(buf3 != NULL);
strncpy(buf3, buf2, end);
buf3[end] = '\0';
retval = new_SerialGraph_t(buf3);
free(buf3);
mrn_dbg(5, mrn_printf(FLF, stderr, "returned sg byte array '%s'\n",
retval->byte_array));
mrn_dbg_func_end();
return retval;
}
Network*
Network::CreateNetworkIN( int argc, char* argv[] )
{
// parse arguments
if( argc >= 5 ) {
const char* phostname = argv[0];
Port pport = (Port)strtoul( argv[1], NULL, 10 );
Rank prank = (Rank)strtoul( argv[2], NULL, 10 );
const char* myhostname = argv[3];
Rank myrank = (Rank)strtoul( argv[4], NULL, 10 );
return new RSHNetwork( phostname, pport, prank,
myhostname, myrank, true );
}
else {
mrn_dbg(1, mrn_printf(FLF, stderr,
"Not enough args, please provide parent/local info\n"));
return NULL;
}
}
Network*
Network::CreateNetworkBE( int argc, char* argv[] )
{
// Get parent/local info from end of BE args
if( argc >= 6 ) {
const char* phostname = argv[argc-5];
Port pport = (Port)strtoul( argv[argc-4], NULL, 10 );
Rank prank = (Rank)strtoul( argv[argc-3], NULL, 10 );
const char* myhostname = argv[argc-2];
Rank myrank = (Rank)strtoul( argv[argc-1], NULL, 10 );
return new RSHNetwork( phostname, pport, prank,
myhostname, myrank, false );
}
else {
mrn_dbg(1, mrn_printf(FLF, stderr,
"Not enough args, please provide parent/local info\n"));
return NULL;
}
}
int SerialGraph_set_Port(SerialGraph_t* sg, char * hostname, Port port, Rank irank)
{
size_t begin_len;
char *old_byte_array, *new_byte_array, *begin_str;
char old_hoststr[256];
char new_hoststr[256];
sprintf(old_hoststr, "[%s:%05hu:%u:", hostname, UnknownPort, irank);
sprintf(new_hoststr, "[%s:%05hu:%u:", hostname, port, irank);
old_byte_array = sg->byte_array;
new_byte_array = (char*) malloc( strlen(old_byte_array) + 10 );
assert(new_byte_array);
// find the old hoststr in the old byte array
begin_str = strstr(old_byte_array, old_hoststr);
if( ! begin_str ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"'%s' not found in the byte_array\n", old_hoststr));
return false;
}
begin_len = begin_str - old_byte_array;
// copy the beginning portion
strncpy(new_byte_array, old_byte_array, begin_len);
new_byte_array[begin_len] = '\0';
// add the updated hoststr
strcat(new_byte_array, new_hoststr);
// concat on the remainder of the old byte array
begin_str += strlen(old_hoststr);
strcat(new_byte_array, begin_str);
// swap the byte arrays
free(old_byte_array);
sg->byte_array = new_byte_array;
return true;
}
void SerialGraph_end_SubTree(SerialGraph_t* sg)
{
size_t curlen, len;
mrn_dbg_func_begin();
curlen = strlen(sg->byte_array);
len = curlen + 2; // appending ']' + '\0'
if( len > sg->arr_len ) {
len *= 2;
sg->byte_array = (char*) realloc(sg->byte_array, len);
if( sg->byte_array == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "realloc(%" PRIszt ") failed\n", len));
exit(0);
}
sg->arr_len = len;
}
sg->byte_array[curlen] = ']';
sg->byte_array[curlen+1] = '\0';
mrn_dbg_func_end();
}
SymReader *MRNetSymbolReaderFactory::openSymbolReader(std::string pathName)
{
const char *pathStr = pathName.c_str();
bool localLib = true;
int tag = PROT_LIB_REQ, ret;
long size;
uint64_t fileContentsLength = 0;
char *fileName = NULL, *fileContents = NULL;
FILE *fp;
PacketPtr packet;
MRNetSymbolReader *msr;
std::map<std::string, MRNetSymbolReader* >::iterator iter;
////GLL comment: This is a temp feature to look for a copy in /tmp RAM disk first
//std::string fileBaseName = basename(pathName.c_str());
//std::string tmpFilePath = "/tmp/" + fileBaseName;
//struct stat fileStat;
//if (stat(tmpFilePath.c_str(), &fileStat) == 0)
//{
//fprintf(stderr, "%s %s\n", pathName.c_str(), tmpFilePath.c_str());
//pathName = tmpFilePath;
//}
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader", gStatOutFp,
"Interposed lib functions called openSymbolReader(%s)\n",
pathName.c_str()));
iter = openReaders_.find(pathName);
if (iter == openReaders_.end())
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader", gStatOutFp,
"no existing reader for %s\n", pathStr));
AsyncGlobalFileStatus myStat(pathStr);
/* TODO: this is a workaround for BlueGene where FGFS is reporting incorrectly */
#ifdef BGL
if (true)
#else
if (IS_YES(myStat.isUnique()))
#endif
{
localLib = false;
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "requesting contents for %s\n", pathStr));
if (stream_->send(tag, "%s", pathStr) == -1)
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "BE: stream::send() failure\n"));
return NULL;
}
if (stream_->flush() == -1)
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "BE: stream::flush() failure\n"));
return NULL;
}
//ret = network_->recv(&tag, packet, &stream_);
ret = stream_->recv(&tag, packet);
if (ret != 1)
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "BE: network::recv() failure\n"));
return NULL;
}
if (tag == PROT_LIB_REQ_ERR)
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "FE reported error sending contents of %s\n", pathStr));
localLib = true;
}
if (tag != PROT_LIB_REQ_RESP)
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "Unexpected tag %d when trying to receive contents of %s\n", tag, pathStr));
localLib = true;
}
#ifdef MRNET40
else if (packet->unpack("%Ac %s", &fileContents, &fileContentsLength, &fileName) == -1)
#else
else if (packet->unpack("%ac %s", &fileContents, &fileContentsLength, &fileName) == -1)
#endif
{
mrn_dbg(2, mrn_printf(__FILE__, __LINE__, "openSymbolReader",
gStatOutFp, "Failed to unpack contents of %s, length %d\n", pathStr, fileContentsLength));
localLib = true;
}
free(fileName);
fileName = NULL;
}
int Message_recv(XPlat_Socket sock_fd, vector_t* packets_in, Rank iinlet_rank)
{
int retval;
unsigned int i;
uint32_t num_packets = 0;
uint32_t num_buffers = 0;
PDR pdrs;
enum pdr_op op = PDR_DECODE;
size_t psz, recv_total, total_bytes = 0;
uint64_t buf_len = 0;
ssize_t readRet;
XPlat_NCBuf_t* ncbufs;
Packet_t* new_packet;
uint64_t *packet_sizes;
char *buf = NULL;
mrn_dbg_func_begin();
mrn_dbg(5, mrn_printf(FLF, stderr, "sock_fd = %d\n", sock_fd));
//
// packet count
//
mrn_dbg(5, mrn_printf(FLF, stderr, "Calling sizeof ...\n"));
pdr_sizeof((pdrproc_t)(pdr_uint32), &num_packets, &buf_len);
assert(buf_len);
buf = (char*) malloc(buf_len + 1);
assert(buf);
mrn_dbg(3, mrn_printf(FLF, stderr, "Reading packet count\n"));
readRet = MRN_recv(sock_fd, buf, buf_len + 1);
if( readRet != (ssize_t)buf_len + 1) {
mrn_dbg(3, mrn_printf(FLF, stderr, "MRN_recv() of packet count failed\n"));
free(buf);
return -1;
}
mrn_dbg(5, mrn_printf(FLF, stderr, "Calling memcreate ...\n"));
pdrmem_create(&pdrs, &(buf[1]), buf_len, op, (pdr_byteorder) buf[0]);
mrn_dbg(5, mrn_printf(FLF, stderr, "Calling uint32 ...\n"));
if( ! pdr_uint32(&pdrs, &num_packets) ) {
error(ERR_PACKING, iinlet_rank, "pdr_uint32() failed\n");
free(buf);
return -1;
}
free(buf);
mrn_dbg(3, mrn_printf(FLF, stderr, "Will receive %u packets\n", num_packets));
if( num_packets > 10000 )
mrn_dbg(1, mrn_printf(FLF, stderr, "WARNING: Receiving more than 10000 packets\n"));
if( num_packets == 0 )
mrn_dbg(1, mrn_printf(FLF, stderr, "WARNING: Receiving zero packets\n"));
num_buffers = num_packets * 2;
//
// packet size vector
//
//buf_len's value is hardcode, breaking pdr encapsulation barrier
buf_len = (sizeof(uint64_t) * num_buffers) + 1; // 1 byte pdr overhead
//mrn_dbg(5, mrn_printf(FLF, stderr, "Calling malloc(%u) ...\n", buf_len));
buf = (char*) malloc( buf_len );
if( buf == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"malloc failed for buf\n"));
return -1;
}
//mrn_dbg(5, mrn_printf(FLF, stderr, "Calling malloc ...\n"));
packet_sizes = (uint64_t*) malloc( sizeof(uint64_t) * (size_t)num_buffers );
if( packet_sizes == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"malloc failed for packet_sizes\n"));
return -1;
}
mrn_dbg(5, mrn_printf(FLF, stderr,
"Calling read(%p, %"PRIu64") for %d buffer lengths.\n",
buf, buf_len, num_buffers));
readRet = MRN_recv(sock_fd, buf, (size_t)buf_len);
if( readRet != (ssize_t)buf_len ) {
mrn_dbg(3, mrn_printf(FLF, stderr, "MRN_recv() %"PRIsszt" of %"PRIsszt" bytes received\n", readRet, buf_len));
free(buf);
//free(packet_sizes);
return -1;
}
pdrmem_create(&pdrs, buf, buf_len, op, pdrmem_getbo());
if( ! pdr_vector(&pdrs, (char*)packet_sizes, num_buffers,
(uint64_t) sizeof(uint64_t), (pdrproc_t)pdr_uint64) ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "pdr_vector() failed\n"));
error(ERR_PACKING, iinlet_rank, "pdr_uint32() failed\n");
free(buf);
free(packet_sizes);
return -1;
}
free(buf);
//
// packets
//
/* recv packet buffers */
ncbufs = (XPlat_NCBuf_t*) malloc( sizeof(XPlat_NCBuf_t) * (size_t)num_buffers );
assert(ncbufs);
mrn_dbg(5, mrn_printf(FLF, stderr, "Reading %u packets:\n", num_packets));
for( i = 0; i < num_buffers; i++ ) {
psz = (size_t) packet_sizes[i];
ncbufs[i].buf = (char*) malloc(psz);
assert(ncbufs[i].buf);
ncbufs[i].len = psz;
total_bytes += psz;
mrn_dbg(5, mrn_printf(FLF, stderr, "- buffer[%u] has size %"PRIsszt"\n",
i, psz));
}
mrn_dbg(5, mrn_printf(FLF, stderr,
"Reading num_buffers=%d\n", num_buffers));
recv_total = 0;
for(i = 0; i < num_buffers; i++ ) {
readRet = MRN_recv( sock_fd, ncbufs[i].buf, ncbufs[i].len );
if( readRet < 0 ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"MRN_recv() failed for packet %u\n", i));
break;
}
recv_total += (int) readRet;
}
if( recv_total != total_bytes ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"%"PRIsszt" of %"PRIsszt" received\n",
recv_total, total_bytes));
for( i = 0; i < num_buffers; i++ )
free((void*)(ncbufs[i].buf));
free(ncbufs);
free(packet_sizes);
return -1;
}
//
// post-processing
//
retval = 0;
mrn_dbg(3, mrn_printf(FLF, stderr, "Creating Packets ...\n"));
for( i = 0; i < num_buffers; i += 2 ) {
mrn_dbg(5, mrn_printf(FLF, stderr, "creating packet[%d]\n",i));
new_packet = new_Packet_t_3( ncbufs[i].len, ncbufs[i].buf,
ncbufs[i+1].len, ncbufs[i+1].buf,
iinlet_rank );
if( new_packet == NULL ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "packet creation failed\n"));
retval = -1;
break;
}
pushBackElement(packets_in, new_packet);
}
// release dynamically allocated memory
// Note: on success return, don't release the packet buffers;
// that memory was passed to the Packet object(s).
if( retval == -1 ) {
for (i = 0; i < num_buffers; i++) {
char * b = ncbufs[i].buf;
free(b);
}
}
free( ncbufs );
free( packet_sizes );
mrn_dbg_func_end();
return retval;
}
int Message_send(Message_t* msg_out, XPlat_Socket sock_fd)
{
uint32_t num_buffers, num_packets;
uint64_t buf_len;
PDR pdrs;
enum pdr_op op = PDR_ENCODE;
int err, go_away = 0;
size_t total_bytes;
ssize_t sret, mcwret;
char* buf = NULL;
XPlat_NCBuf_t ncbufs[2];
uint64_t packet_sizes[2];
mrn_dbg(3, mrn_printf(FLF, stderr, "Sending packets from message %p\n", msg_out));
if( msg_out->packet == NULL ) { // if there is no packet to send
mrn_dbg(3, mrn_printf(FLF, stderr, "Nothing to send!\n"));
return 0;
}
if( (msg_out->packet->tag == PROT_SHUTDOWN) ||
(msg_out->packet->tag == PROT_SHUTDOWN_ACK) )
go_away = 1;
//
// pre-processing
//
/* lightweight send is always single packet */
num_packets = 1;
num_buffers = num_packets * 2;
ncbufs[0].buf = msg_out->packet->hdr;
ncbufs[0].len = msg_out->packet->hdr_len;
ncbufs[1].buf = msg_out->packet->buf;
ncbufs[1].len = (size_t)msg_out->packet->buf_len;
total_bytes = ncbufs[0].len + ncbufs[1].len;
packet_sizes[0] = ncbufs[0].len;
packet_sizes[1] = ncbufs[1].len;
//
// packet count
//
pdr_sizeof((pdrproc_t)(pdr_uint32), &num_packets, &buf_len);
assert(buf_len);
buf = (char*) malloc(buf_len + 1);
assert(buf);
pdrmem_create(&pdrs, &(buf[1]), buf_len, op, pdrmem_getbo());
if( ! pdr_uint32(&pdrs, &num_packets) ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "pdr_uint32() failed\n") );
free(buf);
return -1;
}
buf[0] = (char) pdrmem_getbo();
mrn_dbg(5, mrn_printf(FLF, stderr, "writing packet count\n"));
mcwret = MRN_send(sock_fd, buf, buf_len + 1);
if( mcwret != (ssize_t)buf_len + 1) {
mrn_dbg(1, mrn_printf(FLF, stderr, "MRN_send() failed\n"));
free(buf);
return -1;
}
mrn_dbg(5, mrn_printf(FLF, stderr, "MRN_send() succeeded\n"));
free(buf);
//
// packet size vector
//
buf_len = (num_buffers * sizeof(uint64_t)) + 1; // 1 extra bytes overhead
buf = (char*) malloc( buf_len );
assert(buf);
pdrmem_create(&pdrs, buf, (uint64_t)buf_len, op, pdrmem_getbo());
if( ! pdr_vector(&pdrs, (char*)(packet_sizes), num_buffers,
(uint64_t) sizeof(uint64_t), (pdrproc_t) pdr_uint64) ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "pdr_vector() failed\n"));
free(buf);
return -1;
}
mrn_dbg(5, mrn_printf(FLF, stderr, "write(%p, %d) of size vector\n",
buf, buf_len));
mcwret = MRN_send(sock_fd, buf, (size_t)buf_len);
if( mcwret != (ssize_t)buf_len ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "MRN_send failed\n"));
free(buf);
return -1;
}
free(buf);
//
// packets
//
mrn_dbg(3, mrn_printf(FLF, stderr,
"Sending %d packets (%d total bytes)\n",
num_packets, total_bytes));
sret = XPlat_SocketUtils_Send(sock_fd, ncbufs, num_buffers);
if( sret != (ssize_t)total_bytes ) {
err = XPlat_NetUtils_GetLastError();
mrn_dbg(1, mrn_printf(FLF, stderr,
"XPlat_SocketUtils_Send() returned %d of %d bytes,"
" errno = %d, nbuffers = %d\n",
sret, total_bytes, err, num_packets));
return -1;
}
if( go_away )
XPlat_SocketUtils_Close(sock_fd);
mrn_dbg_func_end();
return 0;
}
BackEndNode_t* new_BackEndNode_t(Network_t* inetwork,
char* imyhostname,
Rank imyrank,
char* iphostname,
Port ipport,
Rank iprank)
{
BackEndNode_t* be;
PeerNode_t* parent;
NetworkTopology_t* nt;
Stream_t* strm;
be = (BackEndNode_t*) calloc( (size_t)1, sizeof(BackEndNode_t) );
assert(be != NULL);
be->network = inetwork;
be->myhostname = strdup(imyhostname);
be->myrank = imyrank;
be->myport = UnknownPort;
be->phostname = strdup(iphostname);
be->pport = ipport;
be->prank = iprank;
be->incarnation = 0;
parent = Network_new_PeerNode(inetwork,
iphostname,
ipport,
iprank,
true,
true);
Network_set_ParentNode(inetwork, parent);
inetwork->local_hostname = imyhostname;
inetwork->local_rank = imyrank;
inetwork->local_back_end_node = be;
nt = new_NetworkTopology_t(inetwork, imyhostname,
(Port)UnknownPort, imyrank, true);
inetwork->network_topology = nt;
// create topology update stream
strm = Network_new_Stream(inetwork, TOPOL_STRM_ID, NULL, 0, 0, 0, 0);
assert( strm != NULL );
// establish data connection with parent
if( ChildNode_init_newChildDataConnection(be, inetwork->parent,
UnknownRank) == -1 ) {
mrn_dbg( 1, mrn_printf(FLF, stderr,
"init_newChildDataConnection() failed\n") );
return NULL;
}
// establish event connection with parent
if( ChildNode_init_newChildEventConnection(be, inetwork->parent) == -1 ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"init_newChildEventConnection() failed\n"));
return NULL;
}
if( ChildNode_send_SubTreeInitDoneReport(be) == -1 ) {
mrn_dbg(1, mrn_printf(FLF, stderr, "ChildNode_send_SubTreeInitDoneReport() failed\n"));
}
return be;
}
int BackEndNode_proc_newStream( BackEndNode_t* be, Packet_t* packet )
{
uint32_t num_backends;
Rank *backends;
unsigned int stream_id;
int tag;
/* Safe since filters are not used in lightweight */
int ds_filter_id = 0, us_filter_id = 0, sync_id = 0;
char* us_filters;
char* sync_filters;
char* ds_filters;
Rank me;
mrn_dbg_func_begin();
tag = packet->tag;
if (tag == PROT_NEW_HETERO_STREAM) {
me = be->network->local_rank;
if (Packet_unpack(packet, "%d %ad %s %s %s",
&stream_id, &backends, &num_backends,
&us_filters, &sync_filters, &ds_filters) == -1) {
mrn_dbg(1, mrn_printf(FLF, stderr, "Packet_unpack() failed\n"));
return -1;
}
if (!Stream_find_FilterAssignment(us_filters, me, us_filter_id)) {
mrn_dbg(3, mrn_printf(FLF, stderr,
"Stream_find_FilterAssignment(upstream) failed, using default\n"));
us_filter_id = TFILTER_NULL;
}
if (!Stream_find_FilterAssignment(ds_filters, me, ds_filter_id)) {
mrn_dbg(3, mrn_printf(FLF, stderr,
"Stream_find_FilterAssignment(downstream) failed, using default\n"));
ds_filter_id = TFILTER_NULL;
}
if (!Stream_find_FilterAssignment(sync_filters, me, sync_id)) {
mrn_dbg(3, mrn_printf(FLF, stderr,
"Stream_find_FilterAssignment(sync) failed, using default\n"));
sync_id = SFILTER_WAITFORALL;
}
}
else { // PROT_NEW_STREAM or PROT_NEW_INTERNAL_STREAM
if (Packet_unpack(packet, "%ud %ad %d %d %d",
&stream_id, &backends,&num_backends,
&us_filter_id, &sync_id, &ds_filter_id) == -1)
{
mrn_dbg(1, mrn_printf(FLF, stderr, "Packet_unpack() failed\n"));
return -1;
}
}
if( TOPOL_STRM_ID != stream_id ) {
Network_new_Stream(be->network, stream_id, backends, num_backends,
us_filter_id, sync_id, ds_filter_id);
}
if( backends != NULL )
free( backends );
if (tag == PROT_NEW_INTERNAL_STREAM) {
// Send ack to parent
if( ! ChildNode_ack_ControlProtocol(be, PROT_NEW_STREAM_ACK, (char)1) ) {
mrn_dbg(1, mrn_printf(FLF, stderr,
"ChildNode_ack_ControlProtocol() failed\n"));
}
}
mrn_dbg_func_end();
return 0;
}
int Message::recv( XPlat_Socket sock_fd, std::list< PacketPtr > &packets_in,
Rank iinlet_rank )
{
Timer t1;
t1.start();
ssize_t retval;
size_t buf_len;
uint64_t *packet_sizes;
char *buf = NULL;
XPlat::SocketUtils::NCBuf* ncbufs;
uint64_t len;
unsigned int i, j;
int rc = 0;
ssize_t total_bytes = 0;
uint32_t num_packets = 0, num_buffers;
PDR pdrs;
enum pdr_op op = PDR_DECODE;
bool using_prealloc = true;
int pkt_size;
PacketPtr pkt;
Stream * strm;
std::list< PacketPtr >::iterator piter;
retval = MRN_recv( sock_fd, _packet_count_buf, size_t(_packet_count_buf_len + 1));
if( retval != (ssize_t)_packet_count_buf_len + 1 ) {
mrn_dbg( 3, mrn_printf(FLF, stderr, "MRN_recv() %" PRIsszt" of %" PRIszt" bytes received\n",
retval, _packet_count_buf_len));
return -1;
}
pdrmem_create( &pdrs, &(_packet_count_buf[1]), _packet_count_buf_len, op, (pdr_byteorder)_packet_count_buf[0] );
if( ! pdr_uint32(&pdrs, &num_packets) ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "pdr_uint32() failed\n") );
return -1;
}
num_buffers = num_packets * 2;
//
// packet size vector
//
// buf_len's value is hardcoded, breaking pdr encapsulation barrier :(
buf_len = (sizeof(uint64_t) * num_buffers) + 1; // 1 byte pdr overhead
if( num_buffers < _ncbuf_len ) {
buf = _packet_sizes_buf;
packet_sizes = _packet_sizes;
ncbufs = _ncbuf;
memset( (void *)_ncbuf, 0, sizeof(_ncbuf) );
}
else {
using_prealloc = false;
buf = (char*) malloc( buf_len );
packet_sizes = (uint64_t*) malloc( sizeof(uint64_t) * num_buffers );
ncbufs = new XPlat::SocketUtils::NCBuf[num_buffers];
memset( (void*)ncbufs, 0,
num_buffers * sizeof(XPlat::SocketUtils::NCBuf) );
}
retval = MRN_recv( sock_fd, buf, buf_len );
if( retval != (ssize_t)buf_len ) {
mrn_dbg( 3, mrn_printf(FLF, stderr, "MRN_recv() %" PRIsszt" of %" PRIsszt" bytes received\n",
retval, buf_len ));
rc = -1;
goto recv_cleanup_return;
}
pdrmem_create( &pdrs, buf, buf_len, op, pdrmem_getbo() );
if( ! pdr_vector(&pdrs, (char*)packet_sizes, num_buffers,
sizeof(uint64_t), (pdrproc_t)pdr_uint64) ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "pdr_vector() failed\n" ));
rc = -1;
goto recv_cleanup_return;
}
/* NOTE: we tell users that packet header and data buffers will have similar
alignment characteristics to malloc, so if we ever stop using malloc
we will need to make sure that the buffers are properly aligned */
for( i = 0; i < num_buffers; i++ ) {
len = packet_sizes[i];
ncbufs[i].buf = (char*) malloc( size_t(len) );
ncbufs[i].len = size_t(len);
total_bytes += size_t(len);
}
retval = XPlat::SocketUtils::Recv( sock_fd, ncbufs, num_buffers );
if( retval != total_bytes ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "SocketUtils::Recv %" PRIsszt" of %" PRIsszt" bytes received\n",
retval, total_bytes) );
rc = -1;
goto recv_cleanup_return;
}
//
// post-processing
//
for( i = 0, j = 0; j < num_packets; i += 2, j++ ) {
PacketPtr new_packet( new Packet((unsigned int)ncbufs[i].len,
ncbufs[i].buf,
ncbufs[i+1].len,
ncbufs[i+1].buf,
iinlet_rank) );
if( new_packet->has_Error() ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "packet creation failed\n") );
rc = -1;
goto recv_cleanup_return;
}
packets_in.push_back( new_packet );
ncbufs[i].buf = NULL;
ncbufs[i+1].buf = NULL;
}
t1.stop();
pkt_size = (int) packets_in.size();
piter = packets_in.begin();
for( ; piter != packets_in.end(); piter++ ) {
pkt = *piter;
strm = _net->get_Stream( pkt->get_StreamId() );
if( strm != NULL ) {
// Time for packet at this point in time.
if( strm->get_PerfData()->is_Enabled(PERFDATA_MET_ELAPSED_SEC,
PERFDATA_CTX_PKT_RECV) ) {
pkt->set_Timer(PERFDATA_PKT_TIMERS_RECV, t1);
}
pkt->start_Timer(PERFDATA_PKT_TIMERS_RECV_TO_FILTER);
pkt->set_IncomingPktCount(pkt_size);
}
}
recv_cleanup_return:
if( -1 == rc ) {
for( unsigned u = 0; u < num_buffers; u++ ) {
if( NULL != ncbufs[u].buf )
free( (void*)(ncbufs[u].buf) );
}
}
if( ! using_prealloc ) {
free( buf );
free( packet_sizes );
delete[] ncbufs;
}
mrn_dbg_func_end();
return rc;
}
int Message::send( XPlat_Socket sock_fd )
{
ssize_t sret;
size_t buf_len, total_bytes = 0;
Stream* strm;
PerfDataMgr* pdm = NULL;
uint64_t *packet_sizes = NULL;
char *buf = NULL;
XPlat::SocketUtils::NCBuf* ncbufs;
Timer tmp;
unsigned int i, j;
int packetLength, rc = 0;
uint32_t num_packets, num_buffers, num_ncbufs;
PDR pdrs;
enum pdr_op op = PDR_ENCODE;
bool using_prealloc = true;
bool go_away = false;
PacketPtr pkt;
std::list< PacketPtr > send_packets;
std::list< PacketPtr >::iterator piter;
_packet_sync.Lock();
_send_sync.Lock();
if( _packets.size() == 0 ) { //nothing to do
mrn_dbg( 3, mrn_printf(FLF, stderr, "Nothing to send!\n") );
_packet_sync.Unlock();
_send_sync.Unlock();
return 0;
}
send_packets = _packets;
_packets.clear();
_packet_sync.Unlock();
piter = send_packets.begin();
for( ; piter != send_packets.end(); piter++ ) {
pkt = *piter;
strm = _net->get_Stream( pkt->get_StreamId() );
if( NULL != strm ) {
pdm = strm->get_PerfData();
if( NULL != pdm ) {
if( pdm->is_Enabled(PERFDATA_MET_ELAPSED_SEC,
PERFDATA_CTX_PKT_SEND) ) {
pkt->start_Timer(PERFDATA_PKT_TIMERS_SEND);
pkt->stop_Timer(PERFDATA_PKT_TIMERS_FILTER_TO_SEND);
}
}
}
}
// Allocation (if required)
num_packets = uint32_t(send_packets.size());
num_buffers = num_packets * 2;
num_ncbufs = num_buffers + 2;
buf_len = ((size_t)num_buffers * sizeof(uint64_t)) + 1; //1 extra bytes overhead
if( num_ncbufs < _ncbuf_len ) {
buf = _packet_sizes_buf;
ncbufs = _ncbuf;
packet_sizes = _packet_sizes;
}
else {
using_prealloc = false;
buf = (char*) malloc( buf_len );
ncbufs = new XPlat::SocketUtils::NCBuf[ num_ncbufs ];
packet_sizes = (uint64_t*) malloc( sizeof(uint64_t) * num_buffers );
}
//
// packets
//
piter = send_packets.begin();
for( i = 0; piter != send_packets.end(); piter++, i += 2 ) {
/* j accounts for skipping first two ncbufs that hold pkt count and sizes */
j = i + 2;
PacketPtr& curPacket = *piter;
/* check for final packet */
int tag = curPacket->get_Tag();
if( (tag == PROT_SHUTDOWN) || (tag == PROT_SHUTDOWN_ACK) )
go_away = true;
uint32_t hsz = curPacket->get_HeaderLen();
uint64_t dsz = curPacket->get_BufferLen();
if( hsz == 0 ) {
/* lazy encoding of packet header */
curPacket->encode_pdr_header();
hsz = curPacket->get_HeaderLen();
}
ncbufs[j].buf = const_cast< char* >( curPacket->get_Header() );
ncbufs[j].len = hsz;
packet_sizes[i] = (uint64_t)hsz;
ncbufs[j+1].buf = const_cast< char* >( curPacket->get_Buffer() );
ncbufs[j+1].len = size_t(dsz);
packet_sizes[i+1] = (uint64_t)dsz;
total_bytes += (size_t)hsz + (size_t)dsz;
}
//
// packet count
//
pdrmem_create( &pdrs, &(_packet_count_buf[1]), _packet_count_buf_len, op, pdrmem_getbo() );
pdr_uint32(&pdrs, &num_packets);
ncbufs[0].buf = _packet_count_buf;
ncbufs[0].len = size_t(_packet_count_buf_len + 1);
_packet_count_buf[0] = (char) pdrmem_getbo();
//
// packet sizes
//
pdrmem_create( &pdrs, buf, buf_len, op, pdrmem_getbo() );
if( ! pdr_vector(&pdrs, (char*)packet_sizes, num_buffers,
sizeof(uint64_t), (pdrproc_t)pdr_uint64) ) {
mrn_dbg( 1, mrn_printf(FLF, stderr, "pdr_vector() failed\n" ));
rc = -1;
goto send_cleanup_return;
}
ncbufs[1].buf = buf;
ncbufs[1].len = buf_len;
// Send it all
sret = XPlat::SocketUtils::Send( sock_fd, ncbufs, num_ncbufs );
if( sret < (ssize_t)(total_bytes + _packet_count_buf_len + buf_len + 1) ) {
mrn_dbg( 1, mrn_printf(FLF, stderr,
"XPlat::SocketUtils::Send() returned %" PRIsszt
" of %" PRIszt" bytes, nbuffers = %u\n",
sret, total_bytes, num_buffers ));
rc = -1;
goto send_cleanup_return;
}
packetLength = (int) send_packets.size();
piter = send_packets.begin();
for( ; piter != send_packets.end(); piter++ ) {
pkt = *piter;
strm = _net->get_Stream( pkt->get_StreamId() );
if( NULL != strm ) {
pdm = strm->get_PerfData();
if( NULL != pdm ) {
pkt->set_Timer( PERFDATA_PKT_TIMERS_RECV_TO_FILTER, tmp );
if( pdm->is_Enabled(PERFDATA_MET_ELAPSED_SEC,
PERFDATA_CTX_PKT_SEND) ) {
pkt->set_OutgoingPktCount( packetLength );
pkt->stop_Timer( PERFDATA_PKT_TIMERS_SEND );
pdm->add_PacketTimers( pkt );
}
else if( pdm->is_Enabled(PERFDATA_MET_ELAPSED_SEC, PERFDATA_CTX_PKT_NET_SENDCHILD) ||
pdm->is_Enabled(PERFDATA_MET_ELAPSED_SEC, PERFDATA_CTX_PKT_NET_SENDPAR) ||
pdm->is_Enabled(PERFDATA_MET_ELAPSED_SEC, PERFDATA_CTX_PKT_FILTER_TO_SEND))
{
pdm->add_PacketTimers( pkt );
}
}
}
}
send_cleanup_return:
_send_sync.Unlock();
if( ! using_prealloc ) {
free( buf );
free( packet_sizes );
delete[] ncbufs;
}
if( go_away ) {
// exit send thread
mrn_dbg( 5, mrn_printf(FLF, stderr, "I'm going away now!\n" ));
tsd_t* tsd = (tsd_t*)XPlat::XPlat_TLSKey->GetUserData();
if( NULL != tsd ) {
delete tsd;
if(XPlat::XPlat_TLSKey->SetUserData(NULL) != 0) {
mrn_dbg(1, mrn_printf(FLF, stderr, "Thread 0x%lx failed to set"
" thread-specific user data to NULL.\n",
XPlat::Thread::GetId()));
}
if(XPlat::XPlat_TLSKey->DestroyData() != 0) {
mrn_dbg(1, mrn_printf(FLF, stderr, "Thread 0x%lx failed to "
"destroy thread-specific data.\n",
XPlat::Thread::GetId()));
}
}
XPlat::Thread::Exit(NULL);
}
mrn_dbg_func_end();
return rc;
}
void Error::perror(const char *str) const
{
mrn_dbg(1, mrn_printf(FLF, stderr, "%s: %s\n", errors[MRN_errno].msg, str) );
}
