/**
* Initialize the LPs with BB capacity and cluster flag then kickoff
* @Params ns node state
* m message
* lp LP
*/
void node_lp_init(node_state * ns, tw_lp * lp) {
burst_buffer_capacity = ((long) (burst_buffer_max_capacity)) * 1000000000;
//burst_buffer_capacity = ((long) (burst_buffer_max_capacity))*10;
//printf("Burst Buffer Capacity:%li\n",burst_buffer_capacity);
printf("In node_lp_init\n");
ns->num_processed = 0;
// nodes are addressed in their logical id space (0...num_client_nodes-1 and
// 0...num_svr_nodes-1, respectively). LPs are computed upon use with
// model-net, other events
ns->id_clust = codes_mapping_get_lp_relative_id(lp->gid, 1, 0);
int id_all = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
// track which cluster we're in
ns->is_in_client = (id_all < num_client_nodes);
ns->is_in_server = (id_all < (num_svr_nodes + num_client_nodes)
&& (id_all >= num_client_nodes));
ns->is_in_bb = (id_all
< (num_svr_nodes + num_client_nodes + num_burst_buffer_nodes)
&& (id_all >= num_svr_nodes + num_client_nodes));
printf("is_in_client=%d\nis_in_svr=%d\nis_in_bb=%d\n", ns->is_in_client,
ns->is_in_server, ns->is_in_bb);
printf("id_all= %d\nnum_client_nodes= %d\n", id_all, num_client_nodes);
// send a self kickoff event
tw_event *e = codes_event_new(lp->gid, codes_local_latency(lp), lp);
node_msg *m = tw_event_data(e);
msg_set_header(node_magic, NODE_KICKOFF, lp->gid, &m->h);
tw_event_send(e);
}
void forwarder_lp_init(forwarder_state * ns, tw_lp * lp) {
//printf("I node_forwarder_lp_init\n");
// like nodes, forwarders in this example are addressed logically
ns->id = codes_mapping_get_lp_relative_id(lp->gid, 1, 0);
int id_all = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
ns->is_in_client = (id_all < num_client_forwarders);
ns->is_in_server = (id_all < (num_svr_forwarders + num_client_forwarders)
&& (id_all >= num_client_forwarders));
ns->is_in_bb = (id_all
< (num_svr_forwarders + num_client_forwarders
+ num_burst_buffer_forwarders)
&& (id_all >= num_svr_forwarders + num_client_forwarders));
}
static void node_finalize(node_state *ns, tw_lp *lp) {
int node_dim_id[3];
int dim_lens[3] = {10, 10, 10};
model_net_torus_get_dim_id(codes_mapping_get_lp_relative_id(lp->gid, 0, 1), 3, dim_lens, node_dim_id); //hard-coded
int dist = find_node_distance(node_dim_id, ns->dest_dim_id);
printf("^ node %llu received %d bytes in %lf seconds, %lf MiB/s sent_count %d recvd_count %d avgLatency %lf sent to a server %d hops away \n",
(unsigned long long)(codes_mapping_get_lp_relative_id(lp->gid,0,1)),
payload_sz*ns->msg_recvd_count,
ns_to_s(ns->end_ts-ns->start_ts),
((double)(payload_sz*num_reqs)/(double)(1024*1024)/ns_to_s(ns->end_ts-ns->start_ts)),
ns->msg_sent_count,
ns->msg_recvd_count,
ns->total_ts/((double)ns->msg_sent_count)/1000.0,
dist);
return;
}
static void s_init(s_state *ns, tw_lp *lp){
ns->mem = 0;
ns->mem_max = 0;
INIT_CODES_CB_INFO(&ns->cb, s_msg, h, tag, c);
ns->id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
tw_event *e = codes_event_new(lp->gid, codes_local_latency(lp), lp);
s_msg *m = tw_event_data(e);
msg_set_header(s_magic, S_KICKOFF, lp->gid, &m->h);
tw_event_send(e);
}
/*Returns the next neighbor to which the packet should be routed by using DOR (Taken from Ning's code of the torus model)*/
static void dimension_order_routing( nodes_state * s,
tw_lpid * dst_lp,
int * dim,
int * dir )
{
int dest[s->params->n_dims];
int dest_id;
/* dummys - check later */
*dim = -1;
*dir = -1;
to_dim_id(codes_mapping_get_lp_relative_id(*dst_lp, 0, 1),
s->params->n_dims, s->params->dim_length, dest);
for(int i = 0; i < s->params->n_dims; i++ )
{
if ( s->dim_position[ i ] - dest[ i ] > s->params->half_length[ i ] )
{
dest_id = s->neighbour_plus_lpID[ i ];
*dim = i;
*dir = 1;
break;
}
if ( s->dim_position[ i ] - dest[ i ] < -s->params->half_length[ i ] )
{
dest_id = s->neighbour_minus_lpID[ i ];
*dim = i;
*dir = 0;
break;
}
if ( ( s->dim_position[i] - dest[i] <= s->params->half_length[i] ) &&
( s->dim_position[ i ] - dest[ i ] > 0 ) )
{
dest_id = s->neighbour_minus_lpID[ i ];
*dim = i;
*dir = 0;
break;
}
if (( s->dim_position[i] - dest[i] >= -s->params->half_length[i] ) &&
( s->dim_position[ i ] - dest[ i ] < 0) )
{
dest_id = s->neighbour_plus_lpID[ i ];
*dim = i;
*dir = 1;
break;
}
}
assert(*dim != -1 && *dir != -1);
*dst_lp = codes_mapping_get_lpid_from_relative(dest_id, NULL, LP_CONFIG_NM,
s->anno, 1);
}
/*Initialize the torus model, this initialization part is borrowed from Ning's torus model */
static void torus_init( nodes_state * s,
tw_lp * lp )
{
int i, j;
char anno[MAX_NAME_LENGTH];
codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL, &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
if (anno[0] == '\0') {
s->anno = NULL;
s->params = &all_params[num_params-1];
}
else {
s->anno = strdup(anno);
int id = configuration_get_annotation_index(anno, anno_map);
s->params = &all_params[id];
}
// shorthand
const torus_param *p = s->params;
s->neighbour_minus_lpID = (int*)malloc(p->n_dims * sizeof(int));
s->neighbour_plus_lpID = (int*)malloc(p->n_dims * sizeof(int));
s->dim_position = (int*)malloc(p->n_dims * sizeof(int));
s->buffer = (int**)malloc(2*p->n_dims * sizeof(int*));
s->next_link_available_time =
(tw_stime**)malloc(2*p->n_dims * sizeof(tw_stime*));
s->next_credit_available_time =
(tw_stime**)malloc(2*p->n_dims * sizeof(tw_stime*));
s->next_flit_generate_time =
(tw_stime**)malloc(2*p->n_dims*sizeof(tw_stime*));
for(i=0; i < 2*p->n_dims; i++)
{
s->buffer[i] = (int*)malloc(p->num_vc * sizeof(int));
s->next_link_available_time[i] =
(tw_stime*)malloc(p->num_vc * sizeof(tw_stime));
s->next_credit_available_time[i] =
(tw_stime*)malloc(p->num_vc * sizeof(tw_stime));
s->next_flit_generate_time[i] =
(tw_stime*)malloc(p->num_vc * sizeof(tw_stime));
}
// calculate my torus coords
to_dim_id(codes_mapping_get_lp_relative_id(lp->gid, 0, 1),
s->params->n_dims, s->params->dim_length, s->dim_position);
/* DEBUG
printf("%lu: my coords:", lp->gid);
for (i = 0; i < p->n_dims; i++)
printf(" %d", s->dim_position[i]);
printf("\n");
*/
int temp_dim_pos[ p->n_dims ];
for ( i = 0; i < p->n_dims; i++ )
temp_dim_pos[ i ] = s->dim_position[ i ];
// calculate minus neighbour's lpID
for ( j = 0; j < p->n_dims; j++ )
{
temp_dim_pos[ j ] = (s->dim_position[ j ] -1 + p->dim_length[ j ]) %
p->dim_length[ j ];
s->neighbour_minus_lpID[j] =
to_flat_id(p->n_dims, p->dim_length, temp_dim_pos);
/* DEBUG
printf(" minus neighbor: flat:%d lpid:%lu\n",
s->neighbour_minus_lpID[j],
codes_mapping_get_lpid_from_relative(s->neighbour_minus_lpID[j],
NULL, LP_CONFIG_NM, s->anno, 1));
*/
temp_dim_pos[ j ] = s->dim_position[ j ];
}
// calculate plus neighbour's lpID
for ( j = 0; j < p->n_dims; j++ )
{
temp_dim_pos[ j ] = ( s->dim_position[ j ] + 1 + p->dim_length[ j ]) %
p->dim_length[ j ];
s->neighbour_plus_lpID[j] =
to_flat_id(p->n_dims, p->dim_length, temp_dim_pos);
/* DEBUG
printf(" plus neighbor: flat:%d lpid:%lu\n",
s->neighbour_plus_lpID[j],
codes_mapping_get_lpid_from_relative(s->neighbour_plus_lpID[j],
NULL, LP_CONFIG_NM, s->anno, 1));
*/
temp_dim_pos[ j ] = s->dim_position[ j ];
}
//printf("\n");
for( j=0; j < 2 * p->n_dims; j++ )
{
for( i = 0; i < p->num_vc; i++ )
{
s->buffer[ j ][ i ] = 0;
s->next_link_available_time[ j ][ i ] = 0.0;
s->next_credit_available_time[j][i] = 0.0;
}
}
// record LP time
s->packet_counter = 0;
torus_collective_init(s, lp);
}
