/**
* @brief Determines how to handle the buffer of event whose send operation
* just finished.
*
* @param[in] me pointer to PE
* @param[in] e pointer to event that we just received
* @param[in] buffer not currently used
*/
static void
send_finish(tw_pe *me, tw_event *e, char * buffer)
{
(void) buffer;
me->stats.s_nsend_network++;
// instrumentation
e->src_lp->kp->kp_stats->s_nsend_network++;
e->src_lp->lp_stats->s_nsend_network++;
if (e->state.owner == TW_net_asend) {
if (e->state.cancel_asend) {
/* Event was cancelled during transmission. We must
* send another message to pass the cancel flag to
* the other node.
*/
e->state.cancel_asend = 0;
e->state.cancel_q = 1;
tw_eventq_push(&outq, e);
} else {
/* Event finished transmission and was not cancelled.
* Add to our sent event queue so we can retain the
* event in case we need to cancel it later. Note it
* is currently in remote format and must be converted
* back to local format for fossil collection.
*/
e->state.owner = TW_pe_sevent_q;
if( g_tw_synchronization_protocol == CONSERVATIVE )
tw_event_free(me, e);
}
return;
}
if (e->state.owner == TW_net_acancel) {
/* We just finished sending the cancellation message
* for this event. We need to free the buffer and
* make it available for reuse.
*/
tw_event_free(me, e);
return;
}
/* Never should happen, not unless we somehow broke this
* module's other functions related to sending an event.
*/
tw_error(
TW_LOC,
"Don't know how to finish send of owner=%u, cancel_q=%d",
e->state.owner,
e->state.cancel_q);
}
static tw_pe * setup_pes(void) {
tw_pe *pe;
tw_pe *master;
int i;
unsigned int num_events_per_pe;
num_events_per_pe = 1 + g_tw_events_per_pe + g_tw_events_per_pe_extra;
master = g_tw_pe[0];
if (!master) {
tw_error(TW_LOC, "No PE configured on this node.");
}
if (g_tw_mynode == g_tw_masternode) {
master->master = 1;
}
master->local_master = 1;
for(i = 0; i < g_tw_npe; i++) {
pe = g_tw_pe[i];
if (g_tw_buddy_alloc) {
g_tw_buddy_master = create_buddy_table(g_tw_buddy_alloc);
if (g_tw_buddy_master == NULL) {
tw_error(TW_LOC, "create_buddy_table() failed.");
}
tw_delta_alloc(pe);
}
pe->pq = tw_pq_create();
tw_eventq_alloc(&pe->free_q, num_events_per_pe);
pe->abort_event = tw_eventq_shift(&pe->free_q);
#ifdef USE_RIO
for (i = 0; i < g_io_events_buffered_per_rank; i++) {
tw_eventq_push(&g_io_free_events, tw_eventq_pop(&g_tw_pe[0]->free_q));
}
#endif
}
if (g_tw_mynode == g_tw_masternode) {
printf("\nROSS Core Configuration: \n");
printf("\t%-50s %11u\n", "Total Nodes", tw_nnodes());
fprintf(g_tw_csv, "%u,", tw_nnodes());
printf("\t%-50s [Nodes (%u) x PE_per_Node (%lu)] %lu\n", "Total Processors", tw_nnodes(), g_tw_npe, (tw_nnodes() * g_tw_npe));
fprintf(g_tw_csv, "%lu,", (tw_nnodes() * g_tw_npe));
printf("\t%-50s [Nodes (%u) x KPs (%lu)] %lu\n", "Total KPs", tw_nnodes(), g_tw_nkp, (tw_nnodes() * g_tw_nkp));
fprintf(g_tw_csv, "%lu,", (tw_nnodes() * g_tw_nkp));
printf("\t%-50s %11llu\n", "Total LPs", (tw_nnodes() * g_tw_npe * g_tw_nlp));
fprintf(g_tw_csv, "%llu,", (tw_nnodes() * g_tw_npe * g_tw_nlp));
printf("\t%-50s %11.2lf\n", "Simulation End Time", g_tw_ts_end);
fprintf(g_tw_csv, "%11.2lf\n", g_tw_ts_end);
switch(g_tw_mapping) {
case LINEAR:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "linear");
fprintf(g_tw_csv, "%s,", "linear");
break;
case ROUND_ROBIN:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "round robin");
fprintf(g_tw_csv, "%s,", "round robin");
break;
case CUSTOM:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "model defined");
fprintf(g_tw_csv, "%s,", "model defined");
break;
}
printf("\n");
#ifndef ROSS_DO_NOT_PRINT
printf("\nROSS Event Memory Allocation:\n");
printf("\t%-50s %11d\n", "Model events", num_events_per_pe);
fprintf(g_tw_csv, "%d,", num_events_per_pe);
printf("\t%-50s %11d\n", "Network events", g_tw_gvt_threshold);
fprintf(g_tw_csv, "%d,", g_tw_gvt_threshold);
printf("\t%-50s %11d\n", "Total events", g_tw_events_per_pe);
fprintf(g_tw_csv, "%d,", g_tw_events_per_pe);
printf("\n");
#endif
}
return master;
}
void tw_event_send(tw_event * event) {
tw_lp *src_lp = event->src_lp;
tw_pe *send_pe = src_lp->pe;
tw_pe *dest_pe = NULL;
tw_peid dest_peid = -1;
tw_stime recv_ts = event->recv_ts;
if (event == send_pe->abort_event) {
if (recv_ts < g_tw_ts_end) {
send_pe->cev_abort = 1;
}
return;
}
//Trap lookahead violations in debug mode
//Note that compiling with the -DNDEBUG flag will turn this off!
if (g_tw_synchronization_protocol == CONSERVATIVE) {
if (recv_ts - tw_now(src_lp) < g_tw_lookahead) {
tw_error(TW_LOC, "Lookahead violation: decrease g_tw_lookahead");
}
}
if (event->out_msgs) {
tw_error(TW_LOC, "It is an error to send an event with pre-loaded output message.");
}
link_causality(event, send_pe->cur_event);
// call LP remote mapping function to get dest_pe
dest_peid = (*src_lp->type->map) ((tw_lpid) event->dest_lp);
if (dest_peid == g_tw_mynode) {
event->dest_lp = tw_getlocal_lp((tw_lpid) event->dest_lp);
dest_pe = event->dest_lp->pe;
if (send_pe == dest_pe && event->dest_lp->kp->last_time <= recv_ts) {
/* Fast case, we are sending to our own PE and there is
* no rollback caused by this send. We cannot have any
* transient messages on local sends so we can return.
*/
tw_pq_enqueue(send_pe->pq, event);
return;
} else {
/* Slower, but still local send, so put into top of
* dest_pe->event_q.
*/
event->state.owner = TW_pe_event_q;
tw_eventq_push(&dest_pe->event_q, event);
if(send_pe != dest_pe) {
send_pe->stats.s_nsend_loc_remote++;
}
}
} else {
/* Slowest approach of all; this is not a local event.
* We need to send it over the network to the other PE
* for processing.
*/
send_pe->stats.s_nsend_net_remote++;
event->state.owner = TW_net_asend;
tw_net_send(event);
}
if(tw_gvt_inprogress(send_pe)) {
send_pe->trans_msg_ts = ROSS_MIN(send_pe->trans_msg_ts, recv_ts);
}
}
/**
* @brief Determines how to handle the newly received event.
*
* @param[in] me pointer to PE
* @param[in] e pointer to event that we just received
* @param[in] buffer not currently used
*/
static void
recv_finish(tw_pe *me, tw_event *e, char * buffer)
{
(void) buffer;
tw_pe *dest_pe;
tw_clock start;
me->stats.s_nread_network++;
me->s_nwhite_recv++;
// printf("recv_finish: remote event [cancel %u] FROM: LP %lu, PE %lu, TO: LP %lu, PE %lu at TS %lf \n",
// e->state.cancel_q, (tw_lpid)e->src_lp, e->send_pe, (tw_lpid)e->dest_lp, me->id, e->recv_ts);
e->dest_lp = tw_getlocal_lp((tw_lpid) e->dest_lp);
dest_pe = e->dest_lp->pe;
// instrumentation
e->dest_lp->kp->kp_stats->s_nread_network++;
e->dest_lp->lp_stats->s_nread_network++;
if(e->send_pe > tw_nnodes()-1)
tw_error(TW_LOC, "bad sendpe_id: %d", e->send_pe);
e->cancel_next = NULL;
e->caused_by_me = NULL;
e->cause_next = NULL;
if(e->recv_ts < me->GVT)
tw_error(TW_LOC, "%d: Received straggler from %d: %lf (%d)",
me->id, e->send_pe, e->recv_ts, e->state.cancel_q);
if(tw_gvt_inprogress(me))
me->trans_msg_ts = ROSS_MIN(me->trans_msg_ts, e->recv_ts);
// if cancel event, retrieve and flush
// else, store in hash table
if(e->state.cancel_q)
{
tw_event *cancel = tw_hash_remove(me->hash_t, e, e->send_pe);
// NOTE: it is possible to cancel the event we
// are currently processing at this PE since this
// MPI module lets me read cancel events during
// event sends over the network.
cancel->state.cancel_q = 1;
cancel->state.remote = 0;
cancel->cancel_next = dest_pe->cancel_q;
dest_pe->cancel_q = cancel;
tw_event_free(me, e);
return;
}
if (g_tw_synchronization_protocol == OPTIMISTIC ||
g_tw_synchronization_protocol == OPTIMISTIC_DEBUG ||
g_tw_synchronization_protocol == OPTIMISTIC_REALTIME ) {
tw_hash_insert(me->hash_t, e, e->send_pe);
e->state.remote = 1;
}
/* NOTE: the final check in the if conditional below was added to make sure
* that we do not execute the fast case unless the cancellation queue is
* empty on the destination PE. Otherwise we need to invoke the normal
* scheduling routines to make sure that a forward event doesn't bypass a
* cancellation event with an earlier timestamp. This is helpful for
* stateful models that produce incorrect results when presented with
* duplicate messages with no rollback between them.
*/
if(me == dest_pe && e->dest_lp->kp->last_time <= e->recv_ts && !dest_pe->cancel_q) {
/* Fast case, we are sending to our own PE and
* there is no rollback caused by this send.
*/
start = tw_clock_read();
tw_pq_enqueue(dest_pe->pq, e);
dest_pe->stats.s_pq += tw_clock_read() - start;
return;
}
if (me->id == dest_pe->id) {
/* Slower, but still local send, so put into top
* of dest_pe->event_q.
*/
e->state.owner = TW_pe_event_q;
tw_eventq_push(&dest_pe->event_q, e);
return;
}
/* Never should happen; MPI should have gotten the
* message to the correct node without needing us
* to redirect the message there for it. This is
* probably a serious bug with the event headers
* not being formatted right.
*/
tw_error(
TW_LOC,
"Event recived by PE %u but meant for PE %u",
me->id,
dest_pe->id);
}
