/**
* IO node send write request to Burst Buffer
* @Params ns node state
* m message
* lp LP
*/
void io_node_send_request(node_state * ns, node_msg * m, tw_lp * lp) {
printf("In Server send REQ\n");
// check that we received the msg from the expected source
// printf("In handle_node_recv_req num_svr_nodes is %d\n",num_svr_nodes);
assert(m->id_clust_src % num_svr_nodes == ns->id_clust);
// setup the response message through the forwarder
forwarder_msg m_fwd;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
m_fwd.dest_node_clust_id = (m->id_clust_src % num_svr_nodes)
% num_burst_buffer_nodes;
//m_fwd.dest_node_clust_id = m->id_clust_src;
m_fwd.node_event_type = NODE_RECV_req; //TO CHANGE WITH BB
//m_fwd.node_event_type = NODE_RECV_ack;
// compute the dest forwarder index, again using a simple modulus
//int dest_fwd_id = ns->id_clust % num_burst_buffer_forwarders;
int dest_fwd_id = ns->id_clust % num_svr_forwarders;
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(dest_fwd_id,
"svr_FORWARDERS", "forwarder", NULL, 0);
ns->pvfs_ts_remote_write += pvfs_tp_write_local_mu;
model_net_event_annotated(net_id_svr, "svr", "req", dest_fwd_lpid,
pvfs_file_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
}
/**
* Burst Buffer send ACK (read operation) to IO node
* @Params ns node state
* m message
* lp LP
*/
void burst_buffer_send_ack(node_state * ns, node_msg * m, tw_lp * lp) {
printf("In Burst Buffer send ACK\n");
// check that we received the msg from the expected source
assert(m->id_clust_src % num_burst_buffer_nodes == ns->id_clust);
// setup the response message through the forwarder
forwarder_msg m_fwd;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
m_fwd.dest_node_clust_id = (m->id_clust_src % num_burst_buffer_nodes)
% num_svr_nodes;
m_fwd.node_event_type = NODE_RECV_ack;
// compute the dest forwarder index, again using a simple modulus
int dest_fwd_id = ns->id_clust % num_svr_forwarders;
// as the relative forwarder IDs are with respect to groups, the group
// name must be used
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(dest_fwd_id,
"bb_FORWARDERS", "forwarder", NULL, 0);
ns->bb_cur_capacity -= pvfs_file_sz;
model_net_event_annotated(net_id_svr, "bb", "ack", dest_fwd_lpid,
pvfs_file_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
}
/**
* Compute node send write request to IO node
* @Params ns node state
* m message
* lp LP
*/
void compute_node_send_request(node_state * ns, node_msg * m, tw_lp * lp) {
printf("In handle_node_next\n");
// we must be in cluster client for this function
assert(ns->is_in_client);
// generate a message to send to the forwarder
forwarder_msg m_fwd;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
// compute the destination in cluster svr to req based on a simple modulo
// of the logical indexes
//printf("num_svr_nodes is %d\n",num_svr_nodes);
m_fwd.dest_node_clust_id = ns->id_clust % num_svr_nodes;
//printf("dest_node_clust_id is %d\n",m_fwd.dest_node_clust_id);
m_fwd.node_event_type = NODE_RECV_req;
// compute the dest forwarder index, again using a simple modulo
//printf("num_client_forwarders is %d\n",num_client_forwarders);
int dest_fwd_id = ns->id_clust % num_client_forwarders;
// printf("dest_node_fwd_id is %d\n",dest_fwd_id);
// as the relative forwarder IDs are with respect to groups, the group
// name must be used
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(dest_fwd_id,
"client_FORWARDERS", "forwarder", NULL, 0);
//printf("dest_fwd_lpid is %d\n",(int)dest_fwd_lpid);
// as cluster nodes have only one network type (+ annotation), no need to
// use annotation-specific messaging
model_net_event_annotated(net_id_client, "client", "req", dest_fwd_lpid,
payload_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
}
void handle_forwarder_recv(forwarder_state * ns, forwarder_msg * m, tw_lp * lp) {
//printf("In handle_forwarder_recv\n");
// compute the node to relay the message to
const char * dest_group;
const char * annotation;
char * category;
int net_id;
if (ns->is_in_client) {
dest_group = "client_CLUSTER";
annotation = "client";
category = "ack";
net_id = net_id_client;
} else if (ns->is_in_server) {
if (m->node_event_type == NODE_RECV_req) {
dest_group = "svr_CLUSTER";
annotation = "svr";
category = "req";
net_id = net_id_svr;
} else if (m->node_event_type == NODE_RECV_ack) {
dest_group = "svr_CLUSTER";
annotation = "svr";
category = "ack";
net_id = net_id_svr;
}
} else if (ns->is_in_bb) {
if (m->node_event_type == NODE_RECV_req) {
dest_group = "bb_CLUSTER";
annotation = "bb";
category = "req";
net_id = net_id_bb;
} else if (m->node_event_type == NODE_RECV_ack) {
dest_group = "bb_CLUSTER";
annotation = "bb";
category = "ack";
net_id = net_id_bb;
}
} else {
dest_group = "storage_CLUSTER";
annotation = "str";
category = "req";
net_id = net_id_storage;
}
tw_lpid dest_lpid = codes_mapping_get_lpid_from_relative(
m->dest_node_clust_id, dest_group, "node", NULL, 0);
node_msg m_node;
msg_set_header(node_magic, m->node_event_type, lp->gid, &m_node.h);
m_node.id_clust_src = m->src_node_clust_id;
// here, we need to use the client or svr cluster's internal network, so we use
// the annotated version of model_net_event
model_net_event_annotated(net_id, annotation, category, dest_lpid,
payload_sz, 0.0, sizeof(m_node), &m_node, 0, NULL, lp);
ns->fwd_forwarder_count++;
}
void handle_forwarder_fwd(forwarder_state * ns, forwarder_msg * m, tw_lp * lp) {
// printf("In handle_forwarder_fwd\n");
// compute the forwarder lpid to forward to
int mod;
const char * dest_group;
char * category;
if (ns->is_in_client) {
//printf("Test Client forwarder \n");
mod = num_svr_forwarders;
dest_group = "svr_FORWARDERS";
category = "req";
} else if (ns->is_in_server) {
if (m->node_event_type == NODE_RECV_ack) {
//printf("Test Server forwarder ACK\n");
mod = num_client_forwarders;
dest_group = "client_FORWARDERS";
category = "ack";
} else if (m->node_event_type == NODE_RECV_req) {
//printf("Test Server forwarder REQ\n");
mod = num_burst_buffer_forwarders;
dest_group = "bb_FORWARDERS";
category = "req";
}
} else if (ns->is_in_bb) {
if (m->node_event_type == NODE_RECV_ack) {
//printf("Test Burst Buffer forwarder ACK\n");
mod = num_svr_forwarders;
dest_group = "svr_FORWARDERS";
category = "ack";
} else if (m->node_event_type == NODE_RECV_req) {
//printf("Test Burst Buffer forwarder REQ\n");
mod = num_storage_forwarders;
dest_group = "storage_FORWARDERS";
category = "req";
}
} else {
//printf("Test Storage forwarder ACK\n");
mod = num_burst_buffer_forwarders;
dest_group = "bb_FORWARDERS";
category = "ack";
}
// compute the ROSS id corresponding to the dest forwarder
//printf("mod is %d\n",mod);
tw_lpid dest_lpid = codes_mapping_get_lpid_from_relative(ns->id % mod,
dest_group, "forwarder", NULL, 0);
forwarder_msg m_fwd = *m;
msg_set_header(forwarder_magic, FORWARDER_RECV, lp->gid, &m_fwd.h);
// here, we need to use the unannotated forwarding network, so we
// use the annotation version of model_net_event
model_net_event_annotated(net_id_forwarding, NULL, category, dest_lpid,
payload_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
ns->fwd_node_count++;
}
/*Returns the next neighbor to which the packet should be routed by using DOR (Taken from Ning's code of the torus model)*/
static void dimension_order_routing( nodes_state * s,
tw_lpid * dst_lp,
int * dim,
int * dir )
{
int dest[s->params->n_dims];
int dest_id;
/* dummys - check later */
*dim = -1;
*dir = -1;
to_dim_id(codes_mapping_get_lp_relative_id(*dst_lp, 0, 1),
s->params->n_dims, s->params->dim_length, dest);
for(int i = 0; i < s->params->n_dims; i++ )
{
if ( s->dim_position[ i ] - dest[ i ] > s->params->half_length[ i ] )
{
dest_id = s->neighbour_plus_lpID[ i ];
*dim = i;
*dir = 1;
break;
}
if ( s->dim_position[ i ] - dest[ i ] < -s->params->half_length[ i ] )
{
dest_id = s->neighbour_minus_lpID[ i ];
*dim = i;
*dir = 0;
break;
}
if ( ( s->dim_position[i] - dest[i] <= s->params->half_length[i] ) &&
( s->dim_position[ i ] - dest[ i ] > 0 ) )
{
dest_id = s->neighbour_minus_lpID[ i ];
*dim = i;
*dir = 0;
break;
}
if (( s->dim_position[i] - dest[i] >= -s->params->half_length[i] ) &&
( s->dim_position[ i ] - dest[ i ] < 0) )
{
dest_id = s->neighbour_plus_lpID[ i ];
*dim = i;
*dir = 1;
break;
}
}
assert(*dim != -1 && *dir != -1);
*dst_lp = codes_mapping_get_lpid_from_relative(dest_id, NULL, LP_CONFIG_NM,
s->anno, 1);
}
/**
* IO node send ACK (read operation) to compute node
* @Params ns node state
* m message
* lp LP
*/
void io_node_send_ack(node_state * ns, node_msg * m, tw_lp * lp) {
printf("In Server send ACK\n");
// setup the response message through the forwarder
forwarder_msg m_fwd;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
m_fwd.dest_node_clust_id = ns->id_clust % num_client_nodes;
m_fwd.node_event_type = NODE_RECV_ack;
// compute the dest forwarder index, again using a simple modulus
int dest_fwd_id = ns->id_clust % num_svr_forwarders;
// as the relative forwarder IDs are with respect to groups, the group
// name must be used
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(dest_fwd_id,
"svr_FORWARDERS", "forwarder", NULL, 0);
ns->pvfs_ts_remote_write += pvfs_tp_write_local_mu;
model_net_event_annotated(net_id_svr, "svr", "ack", dest_fwd_lpid,
payload_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
}
/**
* Burst Buffer send write request to Storage node if it's full
* Otherwise local write + write request to storage later
* @Params ns node state
* m message
* lp LP
*/
void burst_bufer_send_request(node_state * ns, node_msg * m, tw_lp * lp) {
assert(m->id_clust_src % num_burst_buffer_nodes == ns->id_clust);
if (ns->bb_cur_capacity + pvfs_file_sz >= burst_buffer_capacity) { // if full send to storage node
printf("In Burst Buffer send REQ to storage\n");
// setup the response message through the forwarder
forwarder_msg m_fwd;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
m_fwd.dest_node_clust_id = (m->id_clust_src % num_burst_buffer_nodes)
% num_storage_nodes;
m_fwd.node_event_type = NODE_RECV_req;
// compute the dest forwarder index, again using a simple modulus
int dest_fwd_id = ns->id_clust % num_burst_buffer_forwarders;
// as the relative forwarder IDs are with respect to groups, the group
// name must be used
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(
dest_fwd_id, "bb_FORWARDERS", "forwarder", NULL, 0);
ns->pvfs_ts_remote_write += pvfs_tp_write_local_mu;
model_net_event_annotated(net_id_svr, "bb", "req", dest_fwd_lpid,
pvfs_file_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
} else { // if enough room for local write
printf("In Burst Buffer send ACK to Server and REQ to storage later\n");
forwarder_msg m_fwd; //send ack
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd.h);
m_fwd.src_node_clust_id = ns->id_clust;
m_fwd.dest_node_clust_id = (m->id_clust_src % num_burst_buffer_nodes)
% num_svr_nodes;
m_fwd.node_event_type = NODE_RECV_ack;
// compute the dest forwarder index, again using a simple modulus
int dest_fwd_id = ns->id_clust % num_svr_forwarders;
// as the relative forwarder IDs are with respect to groups, the group
// name must be used
tw_lpid dest_fwd_lpid = codes_mapping_get_lpid_from_relative(
dest_fwd_id, "bb_FORWARDERS", "forwarder", NULL, 0);
ns->bb_ts_local_write += burst_buffer_local_mu;
ns->bb_cur_capacity += pvfs_file_sz;
model_net_event_annotated(net_id_svr, "bb", "ack", dest_fwd_lpid,
pvfs_file_sz, 0.0, sizeof(m_fwd), &m_fwd, 0, NULL, lp);
forwarder_msg m_fwd2;
msg_set_header(forwarder_magic, FORWARDER_FWD, lp->gid, &m_fwd2.h);
m_fwd2.src_node_clust_id = ns->id_clust;
m_fwd2.dest_node_clust_id = (m->id_clust_src % num_burst_buffer_nodes)
% num_storage_nodes;
m_fwd2.node_event_type = NODE_RECV_req;
// compute the dest forwarder index, again using a simple modulus
int dest_fwd_id2 = ns->id_clust % num_burst_buffer_forwarders;
tw_lpid dest_fwd_lpid2 = codes_mapping_get_lpid_from_relative(
dest_fwd_id2, "bb_FORWARDERS", "forwarder", NULL, 0);
ns->pvfs_ts_remote_write += pvfs_tp_write_local_mu;
tw_stime offset = tw_rand_integer(lp->rng, 0.0, 5.0);
model_net_event_annotated(net_id_svr, "bb", "req", dest_fwd_lpid2,
pvfs_file_sz, offset, sizeof(m_fwd2), &m_fwd2, 0, NULL, lp);
}
}
