void
tw_gvt_wait(tw_pe * pe)
{
/*
* If new GVT is available grab it and store in
* our PE. Also decrement the global flag so that
* it is known that we have finished doing GVT work.
* Then cleanup our lists by doing a fossil collection.
*/
pe->GVT = sim_gvt;
pe->trans_msg_ts = DBL_MAX;
tw_mutex_lock(&g_tw_gvt_lck);
g_tw_7oclock_node_flag--;
tw_mutex_unlock(&g_tw_gvt_lck);
tw_pe_fossil_collect(pe);
pe->gvt_status = TW_GVT_NORMAL;
#if VERIFY_GVT
printf("%d %d new GVT %lf (gvt_wait)\n",
pe->id, (int) *tw_net_onnode(pe->id), sim_gvt);
printf("%d %d set gvt_flag = %d in gvt_wait\n",
pe->id, (int) *tw_net_onnode(pe->id), g_tw_7oclock_node_flag);
#endif
}
static int
send_begin(tw_pe *me)
{
int changed = 0;
while (posted_sends.cur < send_buffer)
{
tw_event *e = tw_eventq_peek(&outq);
tw_node *dest_node = NULL;
unsigned id = posted_sends.cur;
#if ROSS_MEMORY
tw_event *tmp_prev = NULL;
tw_lp *tmp_lp = NULL;
tw_memory *memory = NULL;
tw_memory *m = NULL;
char *buffer = NULL;
size_t mem_size = 0;
unsigned position = 0;
#endif
if (!e)
break;
if(e == me->abort_event)
tw_error(TW_LOC, "Sending abort event!");
dest_node = tw_net_onnode((*e->src_lp->type->map)
((tw_lpid) e->dest_lp));
//if(!e->state.cancel_q)
//e->event_id = (tw_eventid) ++me->seq_num;
e->send_pe = (tw_peid) g_tw_mynode;
e->send_lp = e->src_lp->gid;
#if ROSS_MEMORY
// pack pointers
tmp_prev = e->prev;
tmp_lp = e->src_lp;
// delete when working
e->src_lp = NULL;
memory = NULL;
if(e->memory)
{
memory = e->memory;
e->memory = (tw_memory *) tw_memory_getsize(me, memory->fd);
e->prev = (tw_event *) memory->fd;
mem_size = (size_t) e->memory;
}
buffer = posted_sends.buffers[id];
memcpy(&buffer[position], e, g_tw_event_msg_sz);
position += g_tw_event_msg_sz;
// restore pointers
e->prev = tmp_prev;
e->src_lp = tmp_lp;
m = NULL;
while(memory)
{
m = memory->next;
if(m)
{
memory->next = (tw_memory *)
tw_memory_getsize(me, m->fd);
memory->fd = m->fd;
}
if(position + mem_size > TW_MEMORY_BUFFER_SIZE)
tw_error(TW_LOC, "Out of buffer space!");
memcpy(&buffer[position], memory, mem_size);
position += mem_size;
memory->nrefs--;
tw_memory_unshift(e->src_lp, memory, memory->fd);
if(NULL != (memory = m))
mem_size = tw_memory_getsize(me, memory->fd);
}
e->memory = NULL;
if (MPI_Isend(buffer,
EVENT_SIZE(e),
MPI_BYTE,
*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#else
if (MPI_Isend(e,
(int)EVENT_SIZE(e),
MPI_BYTE,
(int)*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#endif
tw_eventq_pop(&outq);
e->state.owner = e->state.cancel_q
? TW_net_acancel
: TW_net_asend;
posted_sends.event_list[id] = e;
posted_sends.cur++;
me->s_nwhite_sent++;
changed = 1;
}
return changed;
}
static void
tw_gvt_compute(tw_pe * pe)
{
tw_stime gvt_temp;
tw_stime trans_temp;
tw_pe *p;
pe->LVT = min(tw_pq_minimum(pe->pq), pe->trans_msg_ts);
#if 0
printf("%d pq %lf, trans %lf \n", pe->id, tw_pq_minimum(pe->pq), pe->trans_msg_ts);
#endif
if (pe->LVT < sim_gvt)
tw_error(
TW_LOC,
"%d: LVT < GVT!!! gvt: %f lvt: %f t_msg: %f",
pe->id, sim_gvt, pe->LVT, pe->trans_msg_ts);
tw_mutex_lock(&g_tw_gvt_lck);
if (g_tw_7oclock_node_flag != 1) {
g_tw_7oclock_node_flag--;
tw_mutex_unlock(&g_tw_gvt_lck);
pe->gvt_status = TW_GVT_WAIT_REMOTE;
return;
}
/* Last to compute GVT for this node (LGVT) */
gvt_temp = pe->LVT;
trans_temp = pe->trans_msg_ts;
p = NULL;
while ((p = tw_pe_next(p)))
{
if (gvt_temp > p->LVT)
gvt_temp = p->LVT;
if (trans_temp > p->trans_msg_ts)
trans_temp = p->trans_msg_ts;
}
if (trans_temp < gvt_temp)
gvt_temp = trans_temp;
if(TW_DISTRIBUTED) {
node_lvt = gvt_temp;
g_tw_7oclock_node_flag--;
} else {
g_tw_gvt_done++;
sim_gvt = gvt_temp;
g_tw_7oclock_node_flag = -1;
}
tw_mutex_unlock(&g_tw_gvt_lck);
#if VERIFY_GVT || 0
if(!TW_DISTRIBUTED)
{
printf("%d %d new GVT %f in gvt_compute\n",
pe->id, (int) *tw_net_onnode(pe->id), pe->GVT);
} else
printf("%d %d new LVT %f in gvt_compute\n",
pe->id, (int) *tw_net_onnode(pe->id), pe->LVT);
printf("%d %d sets gvt flag = %d in gvt_compute \n",
pe->id, (int) *tw_net_onnode(pe->id), g_tw_7oclock_node_flag);
#endif
/* Send LVT to master node on network */
if (!tw_node_eq(&pe->node, &g_tw_masternode))
tw_net_send_lvt(pe, node_lvt);
pe->gvt_status = TW_GVT_WAIT_REMOTE;
pe->trans_msg_ts = DBL_MAX;
if(TW_PARALLEL)
{
if (sim_gvt != pe->GVT) {
g_tw_gvt_no_change = 0;
} else {
g_tw_gvt_no_change++;
if (g_tw_gvt_no_change >= g_tw_gvt_max_no_change) {
tw_error(
TW_LOC,
"GVT computed %d times in a row"
" without changing: GVT = %g"
" -- GLOBAL SYNCH -- out of memory!",
g_tw_gvt_no_change, sim_gvt);
}
}
pe->GVT = sim_gvt;
if (sim_gvt > g_tw_ts_end)
return;
tw_pe_fossil_collect(pe);
pe->gvt_status = TW_GVT_NORMAL;
pe->trans_msg_ts = DBL_MAX;
}
}
void
tw_gvt_step1(tw_pe * me)
{
if(TW_PARALLEL)
{
if (me->master)
{
gvt_cnt++;
if (gvt_cnt >= g_tw_gvt_interval &&
me->gvt_status == TW_GVT_NORMAL)
{
gvt_cnt = 0;
if (g_tw_7oclock_node_flag == -g_tw_npe)
{
tw_mutex_lock(&g_tw_gvt_lck);
g_tw_7oclock_node_flag = g_tw_npe;
tw_mutex_unlock(&g_tw_gvt_lck);
me->gvt_status = TW_GVT_COMPUTE;
}
}
} else if (g_tw_7oclock_node_flag > 0
&& me->gvt_status == TW_GVT_NORMAL)
{
me->gvt_status = TW_GVT_COMPUTE;
}
return;
}
if(TW_DISTRIBUTED)
{
if (me->gvt_status != TW_GVT_NORMAL)
return;
if (me->local_master)
{
/*
* start GVT in motion, all nodes see this simultaneously
* and make a consistent cut by setting the gvt flag
* atomically over the network
*/
if (tw_clock_now(me) >= g_tw_clock_gvt_interval)
{
if (g_tw_7oclock_node_flag == -g_tw_npe)
{
gvt_cnt++;
g_tw_clock_gvt_interval += g_tw_clock_gvt_window_size;
g_tw_7oclock_node_flag = g_tw_npe;
me->gvt_status = TW_GVT_COMPUTE_LGVT;
#if VERIFY_GVT || 0
printf("\n\n\nGVT COUNT = %d (%d %d) \n",
gvt_cnt, me->id, (int) *tw_net_onnode(me->id));
#endif
}
}
} else
{
if (g_tw_7oclock_node_flag > 0 &&
tw_clock_now(me) >= g_tw_clock_gvt_interval)
{
me->gvt_status = TW_GVT_COMPUTE_LGVT;
}
}
}
}