void
packet_send(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_lp * lp)
{
int i;
tw_lpid dst_lp;
tw_stime ts;
tw_event *e;
nodes_message *m;
ts = LINK_DELAY+msg->transmission_time;
/*
* Routing in the torus, start from the first dimension
*/
if (lp->gid != msg->dest_lp )
{
bf->c3 = 1;
dimension_order_routing(s,msg,&dst_lp);
msg->source_dim = s->source_dim;
msg->source_direction = s->direction;
}
else
{
bf->c3 = 0;
dst_lp = lp->gid;
}
e = tw_event_new(dst_lp, ts, lp);
m = tw_event_data(e);
m->type = ARRIVAL;
// Carry on the message info
for( i = 0; i < N_dims; i++ )
m->dest[i] = msg->dest[i];
m->dest_lp = msg->dest_lp;
m->transmission_time = msg->transmission_time;
m->source_dim = msg->source_dim;
m->source_direction = msg->source_direction;
m->packet_ID = msg->packet_ID;
m->travel_start_time = msg->travel_start_time;
m->my_N_hop = msg->my_N_hop;
m->my_N_queue = msg->my_N_queue;
m->queueing_times = msg->queueing_times;
tw_event_send(e);
}
/* send a packet from one torus node to another torus node.
A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q */
void
packet_send( nodes_state * s,
tw_bf * bf,
nodes_message * msg,
tw_lp * lp )
{
bf->c3 = 0;
bf->c1 = 0;
int i, vc = 0, tmp_dir, tmp_dim;
tw_stime ts;
tw_event *e;
nodes_message *m;
tw_lpid dst_lp = msg->dest_lp;
int tokens_min = 0;
if( msg->next_stop == -1 )
{
/* no destination has been set */
dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir );
}
else
{
/* destination has been set and this is a waiting message */
dst_lp = msg->next_stop;
if(msg->wait_type == -1)
{
tmp_dir = msg->source_direction;
tmp_dim = msg->source_dim;
}
else
{
tmp_dim = msg->wait_dim;
tmp_dir = msg->wait_dir;
}
}
/* if buffer space is not available then add message in the waiting queue */
if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] >= num_buf_slots * num_chunks)
{
// re-schedule the message in the future
ts = 0.1 + tw_rand_exponential( lp->rng, MEAN_INTERVAL/200);
e = tw_event_new( lp->gid, ts, lp );
m = tw_event_data( e );
m->wait_type = SEND;
m->type = WAIT;
m->wait_dim = tmp_dim;
m->wait_dir = tmp_dir;
m->next_stop = dst_lp;
m->chunk_id = msg->chunk_id;
m->dest_lp = msg->dest_lp;
// added may 20
m->source_direction = msg->source_direction;
m->source_dim = msg->source_dim;
m->packet_ID = msg->packet_ID;
m->sender_lp = msg->sender_lp;
for (i=0; i < N_dims; i++)
m->dest[i] = msg->dest[i];
tw_event_send(e);
}
else
{
bf->c3 = 1;
msg->saved_src_dir = tmp_dir;
msg->saved_src_dim = tmp_dim;
ts = tw_rand_exponential( lp->rng, ( double )head_delay/200 )+ head_delay;
// For reverse computation
msg->saved_available_time = s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0];
s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] = max( s->next_link_available_time[ tmp_dir + ( tmp_dim * 2 )][0], tw_now(lp) );
s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] += ts;
e = tw_event_new( dst_lp, s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] - tw_now(lp), lp );
m = tw_event_data( e );
m->type = ARRIVAL;
//Carry on the message info
m->source_dim = tmp_dim;
m->source_direction = tmp_dir;
m->next_stop = dst_lp;
m->sender_lp = lp->gid;
m->chunk_id = msg->chunk_id;
for( i = 0; i < N_dims; i++ )
m->dest[ i ] = msg->dest[ i ];
m->dest_lp = msg->dest_lp;
m->packet_ID = msg->packet_ID;
m->travel_start_time = msg->travel_start_time;
m->my_N_hop = msg->my_N_hop;
tw_event_send( e );
s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ]++;
if(msg->chunk_id == num_chunks - 1 && msg->sender_lp == -1)
{
bf->c1 = 1;
int index = floor( N_COLLECT_POINTS * ( tw_now( lp ) / g_tw_ts_end ) );
N_generated_storage[ index ]++;
}
} // end else
}
/*Generates a packet. Queue space is checked first and a packet is injected
* in the queue if space is available. */
void
packet_generate( nodes_state * s,
tw_bf * bf,
nodes_message * msg,
tw_lp * lp )
{
int i, j, tmp_dir=-1, tmp_dim=-1;
tw_stime ts;
/* event triggered when packet head is sent */
tw_event * e_h;
nodes_message *m;
/* for NN traffic, we check for exact neighbors based on the torus dimensions
* thats why we decide NN's here not at the MPI LP level */
if(TRAFFIC == NEAREST_NEIGHBOR)
{
int dest_counter = msg->dest_lp;
if( dest_counter < N_dims_sim)
msg->dest_lp = s->neighbour_minus_lpID_sim[dest_counter];
else if(dest_counter >= N_dims_sim && dest_counter < 2 * N_dims_sim)
msg->dest_lp = s->neighbour_plus_lpID_sim[dest_counter-N_dims_sim];
}
/* again for the diagonal traffic, we calculate destinations at the torus LP level not
* the MPI level as we need information about torus coordinates */
if(TRAFFIC == DIAGONAL)
{
msg->dest_lp = 0;
int dest[N_dims_sim];
for( i = 0; i < N_dims_sim; i++ )
{
dest[i] = dim_length_sim[i] - s->dim_position_sim[i] -1;
msg->dest_lp += factor_sim[ i ] * dest[ i ];
}
if( lp->gid == TRACK_LP )
{
printf("\n LP GID %d sending message to ", (int)lp->gid);
for( j = 0; j < N_dims_sim; j++ )
printf(" %d ", dest[ j ] );
printf("\n ");
}
}
tw_lpid dst_lp = msg->dest_lp;
dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir );
if(tmp_dir == -1 || tmp_dim == -1)
{
printf("\n LP %d dest LP %d dim %d dir %d ", (int)lp->gid, (int)msg->dest_lp, tmp_dim, tmp_dir);
assert( 0 );
}
for(j = 0; j < num_chunks; j++)
{
/* adding a small constant to prevent zero time-stamps */
ts = 0.1 + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200);
e_h = tw_event_new( lp->gid, ts, lp);
msg->source_direction = tmp_dir;
msg->source_dim = tmp_dim;
m = tw_event_data( e_h );
m->next_stop = dst_lp;
m->dest_lp = msg->dest_lp;
m->travel_start_time = msg->travel_start_time;
m->packet_ID = msg->packet_ID;
m->chunk_id = j;
m->sender_lp = -1;
int dim_N[ N_dims ];
dim_N[ 0 ] = m->dest_lp;
// find destination dimensions using destination LP ID
for (i=0; i < N_dims; i++)
{
m->dest[ i ] = dim_N[ i ] % dim_length[ i ];
dim_N[ i + 1 ] = ( dim_N[ i ] - m->dest[ i ] ) / dim_length[ i ];
}
if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < num_buf_slots * num_chunks)
{
m->my_N_hop = 0;
m->wait_type = -1;
// Send the packet out
m->type = SEND;
m->source_direction = tmp_dir;
m->source_dim = tmp_dim;
#if DEBUG
/*if(m->packet_ID == TRACK)
printf("\n (%lld) Packet generated %lld
Buffer space %d
time %lf tmp_dir %d tmp_dim %d
num_chunks %d dest_lp %lld",
lp->gid, m->packet_ID,
s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ],
tw_now(lp), tmp_dir, tmp_dim,
num_chunks, msg->dest_lp );*/
#endif
tw_event_send(e_h);
}
else
{
printf("\n %d Packet queued in line, dir %d dim %d buffer space %d dest LP %d wait type %d ",
(int)lp->gid,
tmp_dir,
tmp_dim,
s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ],
(int)msg->dest_lp,
msg->wait_type);
exit(-1);
}
}
}
/* send a packet from one torus node to another torus node
A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q */
static void packet_send( nodes_state * s,
tw_bf * bf,
nodes_message * msg,
tw_lp * lp )
{
bf->c2 = 0;
bf->c1 = 0;
int tmp_dir, tmp_dim;
tw_stime ts;
tw_event *e;
nodes_message *m;
tw_lpid dst_lp = msg->dest_lp;
dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir );
if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < s->params->buffer_size)
{
bf->c2 = 1;
msg->saved_src_dir = tmp_dir;
msg->saved_src_dim = tmp_dim;
ts = tw_rand_exponential( lp->rng, s->params->head_delay/200.0 ) +
s->params->head_delay;
// For reverse computation
msg->saved_available_time = s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0];
s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] = maxd( s->next_link_available_time[ tmp_dir + ( tmp_dim * 2 )][0], tw_now(lp) );
s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] += ts;
//e = tw_event_new( dst_lp, s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] - tw_now(lp), lp );
//m = tw_event_data( e );
//memcpy(m, msg, torus_get_msg_sz() + msg->remote_event_size_bytes);
void * m_data;
e = model_net_method_event_new(dst_lp,
s->next_link_available_time[tmp_dir+(tmp_dim*2)][0] - tw_now(lp),
lp, TORUS, (void**)&m, &m_data);
memcpy(m, msg, sizeof(nodes_message));
if (msg->remote_event_size_bytes) {
memcpy(m_data, model_net_method_get_edata(TORUS, msg),
msg->remote_event_size_bytes);
}
m->type = ARRIVAL;
if(msg->packet_ID == TRACE)
printf("\n lp %d packet %lld flit id %d being sent to %d after time %lf ", (int) lp->gid, msg->packet_ID, msg->chunk_id, (int)dst_lp, s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0] - tw_now(lp));
//Carry on the message info
m->source_dim = tmp_dim;
m->source_direction = tmp_dir;
m->next_stop = dst_lp;
m->sender_node = lp->gid;
m->local_event_size_bytes = 0; /* We just deliver the local event here */
tw_event_send( e );
s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ]++;
uint64_t num_chunks = msg->packet_size/s->params->chunk_size;
if(msg->packet_size % s->params->chunk_size)
num_chunks++;
if(msg->chunk_id == num_chunks - 1)
{
bf->c1 = 1;
/* Invoke an event on the sending server */
if(msg->local_event_size_bytes > 0)
{
tw_event* e_new;
nodes_message* m_new;
void* local_event;
ts = (1/s->params->link_bandwidth) * msg->local_event_size_bytes;
e_new = tw_event_new(msg->sender_svr, ts, lp);
m_new = tw_event_data(e_new);
//local_event = (char*)msg;
//local_event += torus_get_msg_sz() + msg->remote_event_size_bytes;
local_event = (char*)model_net_method_get_edata(TORUS, msg) +
msg->remote_event_size_bytes;
memcpy(m_new, local_event, msg->local_event_size_bytes);
tw_event_send(e_new);
}
}
} // end if
else
{
printf("\n buffer overflown ");
MPI_Finalize();
exit(-1);
}
}
/*Generates a packet. If there is a buffer slot available, then the packet is
injected in the network. Else, a buffer overflow exception is thrown.
TODO: We might want to modify this so that if the buffer is full, the packet
injection is delayed in turn slowing down the injection rate. The average achieved
injection rate can be reported at the end of the simulation. */
static void packet_generate( nodes_state * s,
tw_bf * bf,
nodes_message * msg,
tw_lp * lp )
{
// printf("\n msg local event size %d remote event size %d ", msg->local_event_size_bytes, msg->remote_event_size_bytes);
int j, tmp_dir=-1, tmp_dim=-1, total_event_size;
tw_stime ts;
// event triggered when packet head is sent
tw_event * e_h;
nodes_message *m;
tw_lpid dst_lp;
// TODO: be annotation-aware
dst_lp = model_net_find_local_device(TORUS, s->anno, 0,
msg->final_dest_gid);
//mapping_offset, &dst_lp);
// dest_lp gets included to other required msgs through memcpys, so just
// set here
msg->dest_lp = dst_lp;
dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir );
msg->saved_src_dim = tmp_dim;
msg->saved_src_dir = tmp_dir;
//msg->saved_available_time = s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0];
msg->travel_start_time = tw_now(lp);
msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
msg->my_N_hop = 0;
uint64_t num_chunks = msg->packet_size/s->params->chunk_size;
if(msg->packet_size % s->params->chunk_size)
num_chunks++;
s->packet_counter++;
if(msg->packet_ID == TRACE)
printf("\n packet generated %lld at lp %d dest %d final dest %d", msg->packet_ID, (int)lp->gid, (int)dst_lp, (int)msg->dest_lp);
for(j = 0; j < num_chunks; j++)
{
if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < s->params->buffer_size)
{
ts = j + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200);
//s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0] = max(s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0], tw_now(lp));
//s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0] += ts;
//e_h = tw_event_new( lp->gid, s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0] - tw_now(lp), lp);
//e_h = tw_event_new(lp->gid, ts, lp);
msg->source_direction = tmp_dir;
msg->source_dim = tmp_dim;
void *m_data;
e_h = model_net_method_event_new(lp->gid, ts, lp, TORUS, (void**)&m,
(void**)&m_data);
//m = tw_event_data( e_h );
//memcpy(m, msg, torus_get_msg_sz() + msg->local_event_size_bytes + msg->remote_event_size_bytes);
void *m_data_src = model_net_method_get_edata(TORUS, msg);
memcpy(m, msg, sizeof(nodes_message));
if (msg->remote_event_size_bytes) {
memcpy(m_data, m_data_src,
msg->remote_event_size_bytes);
m_data = (char*)m_data + msg->remote_event_size_bytes;
m_data_src = (char*)m_data_src + msg->remote_event_size_bytes;
}
if (msg->local_event_size_bytes) {
memcpy(m_data, m_data_src, msg->local_event_size_bytes);
}
m->next_stop = dst_lp;
m->chunk_id = j;
// find destination dimensions using destination LP ID
m->type = SEND;
m->source_direction = tmp_dir;
m->source_dim = tmp_dim;
tw_event_send(e_h);
}
else
{
printf("\n %d Packet queued in line increase buffer space, dir %d dim %d buffer space %d dest LP %d ", (int)lp->gid, tmp_dir, tmp_dim, s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ], (int)msg->dest_lp);
MPI_Finalize();
exit(-1);
}
}
total_event_size = model_net_get_msg_sz(TORUS) + msg->remote_event_size_bytes + msg->local_event_size_bytes;
/* record the statistics of the generated packets */
mn_stats* stat;
stat = model_net_find_stats(msg->category, s->torus_stats_array);
stat->send_count++;
stat->send_bytes += msg->packet_size;
stat->send_time += (1/s->params->link_bandwidth) * msg->packet_size;
/* record the maximum ROSS event size */
if(stat->max_event_size < total_event_size)
stat->max_event_size = total_event_size;
}
