/**
* @brief If there are any openings in the posted_sends queue, start sends
* for events in the outgoing queue.
*
* @param[in] me pointer to the PE
* @return 0 if no changes are made to the posted_sends queue, 1 otherwise.
*/
static int
send_begin(tw_pe *me)
{
int changed = 0;
while (posted_sends.cur < send_buffer)
{
tw_event *e = tw_eventq_peek(&outq);
tw_peid dest_pe;
unsigned id = posted_sends.cur;
if (!e)
break;
if(e == me->abort_event)
tw_error(TW_LOC, "Sending abort event!");
dest_pe = (*e->src_lp->type->map) ((tw_lpid) e->dest_lp);
e->send_pe = (tw_peid) g_tw_mynode;
e->send_lp = e->src_lp->gid;
if (MPI_Isend(e,
(int)EVENT_SIZE(e),
MPI_BYTE,
(int)dest_pe,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
tw_eventq_pop(&outq);
e->state.owner = e->state.cancel_q
? TW_net_acancel
: TW_net_asend;
posted_sends.event_list[id] = e;
posted_sends.cur++;
me->s_nwhite_sent++;
changed = 1;
}
return changed;
}
static tw_pe * setup_pes(void) {
tw_pe *pe;
tw_pe *master;
int i;
unsigned int num_events_per_pe;
num_events_per_pe = 1 + g_tw_events_per_pe + g_tw_events_per_pe_extra;
master = g_tw_pe[0];
if (!master) {
tw_error(TW_LOC, "No PE configured on this node.");
}
if (g_tw_mynode == g_tw_masternode) {
master->master = 1;
}
master->local_master = 1;
for(i = 0; i < g_tw_npe; i++) {
pe = g_tw_pe[i];
if (g_tw_buddy_alloc) {
g_tw_buddy_master = create_buddy_table(g_tw_buddy_alloc);
if (g_tw_buddy_master == NULL) {
tw_error(TW_LOC, "create_buddy_table() failed.");
}
tw_delta_alloc(pe);
}
pe->pq = tw_pq_create();
tw_eventq_alloc(&pe->free_q, num_events_per_pe);
pe->abort_event = tw_eventq_shift(&pe->free_q);
#ifdef USE_RIO
for (i = 0; i < g_io_events_buffered_per_rank; i++) {
tw_eventq_push(&g_io_free_events, tw_eventq_pop(&g_tw_pe[0]->free_q));
}
#endif
}
if (g_tw_mynode == g_tw_masternode) {
printf("\nROSS Core Configuration: \n");
printf("\t%-50s %11u\n", "Total Nodes", tw_nnodes());
fprintf(g_tw_csv, "%u,", tw_nnodes());
printf("\t%-50s [Nodes (%u) x PE_per_Node (%lu)] %lu\n", "Total Processors", tw_nnodes(), g_tw_npe, (tw_nnodes() * g_tw_npe));
fprintf(g_tw_csv, "%lu,", (tw_nnodes() * g_tw_npe));
printf("\t%-50s [Nodes (%u) x KPs (%lu)] %lu\n", "Total KPs", tw_nnodes(), g_tw_nkp, (tw_nnodes() * g_tw_nkp));
fprintf(g_tw_csv, "%lu,", (tw_nnodes() * g_tw_nkp));
printf("\t%-50s %11llu\n", "Total LPs", (tw_nnodes() * g_tw_npe * g_tw_nlp));
fprintf(g_tw_csv, "%llu,", (tw_nnodes() * g_tw_npe * g_tw_nlp));
printf("\t%-50s %11.2lf\n", "Simulation End Time", g_tw_ts_end);
fprintf(g_tw_csv, "%11.2lf\n", g_tw_ts_end);
switch(g_tw_mapping) {
case LINEAR:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "linear");
fprintf(g_tw_csv, "%s,", "linear");
break;
case ROUND_ROBIN:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "round robin");
fprintf(g_tw_csv, "%s,", "round robin");
break;
case CUSTOM:
printf("\t%-50s %11s\n", "LP-to-PE Mapping", "model defined");
fprintf(g_tw_csv, "%s,", "model defined");
break;
}
printf("\n");
#ifndef ROSS_DO_NOT_PRINT
printf("\nROSS Event Memory Allocation:\n");
printf("\t%-50s %11d\n", "Model events", num_events_per_pe);
fprintf(g_tw_csv, "%d,", num_events_per_pe);
printf("\t%-50s %11d\n", "Network events", g_tw_gvt_threshold);
fprintf(g_tw_csv, "%d,", g_tw_gvt_threshold);
printf("\t%-50s %11d\n", "Total events", g_tw_events_per_pe);
fprintf(g_tw_csv, "%d,", g_tw_events_per_pe);
printf("\n");
#endif
}
return master;
}
static int
send_begin(tw_pe *me)
{
int changed = 0;
while (posted_sends.cur < send_buffer)
{
tw_event *e = tw_eventq_peek(&outq);
tw_node *dest_node = NULL;
unsigned id = posted_sends.cur;
#if ROSS_MEMORY
tw_event *tmp_prev = NULL;
tw_lp *tmp_lp = NULL;
tw_memory *memory = NULL;
tw_memory *m = NULL;
char *buffer = NULL;
size_t mem_size = 0;
unsigned position = 0;
#endif
if (!e)
break;
if(e == me->abort_event)
tw_error(TW_LOC, "Sending abort event!");
dest_node = tw_net_onnode((*e->src_lp->type->map)
((tw_lpid) e->dest_lp));
//if(!e->state.cancel_q)
//e->event_id = (tw_eventid) ++me->seq_num;
e->send_pe = (tw_peid) g_tw_mynode;
e->send_lp = e->src_lp->gid;
#if ROSS_MEMORY
// pack pointers
tmp_prev = e->prev;
tmp_lp = e->src_lp;
// delete when working
e->src_lp = NULL;
memory = NULL;
if(e->memory)
{
memory = e->memory;
e->memory = (tw_memory *) tw_memory_getsize(me, memory->fd);
e->prev = (tw_event *) memory->fd;
mem_size = (size_t) e->memory;
}
buffer = posted_sends.buffers[id];
memcpy(&buffer[position], e, g_tw_event_msg_sz);
position += g_tw_event_msg_sz;
// restore pointers
e->prev = tmp_prev;
e->src_lp = tmp_lp;
m = NULL;
while(memory)
{
m = memory->next;
if(m)
{
memory->next = (tw_memory *)
tw_memory_getsize(me, m->fd);
memory->fd = m->fd;
}
if(position + mem_size > TW_MEMORY_BUFFER_SIZE)
tw_error(TW_LOC, "Out of buffer space!");
memcpy(&buffer[position], memory, mem_size);
position += mem_size;
memory->nrefs--;
tw_memory_unshift(e->src_lp, memory, memory->fd);
if(NULL != (memory = m))
mem_size = tw_memory_getsize(me, memory->fd);
}
e->memory = NULL;
if (MPI_Isend(buffer,
EVENT_SIZE(e),
MPI_BYTE,
*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#else
if (MPI_Isend(e,
(int)EVENT_SIZE(e),
MPI_BYTE,
(int)*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#endif
tw_eventq_pop(&outq);
e->state.owner = e->state.cancel_q
? TW_net_acancel
: TW_net_asend;
posted_sends.event_list[id] = e;
posted_sends.cur++;
me->s_nwhite_sent++;
changed = 1;
}
return changed;
}
