static int
send_begin(tw_pe *me)
{
int changed = 0;
while (posted_sends.cur < send_buffer)
{
tw_event *e = tw_eventq_peek(&outq);
tw_node *dest_node = NULL;
unsigned id = posted_sends.cur;
#if ROSS_MEMORY
tw_event *tmp_prev = NULL;
tw_lp *tmp_lp = NULL;
tw_memory *memory = NULL;
tw_memory *m = NULL;
char *buffer = NULL;
size_t mem_size = 0;
unsigned position = 0;
#endif
if (!e)
break;
if(e == me->abort_event)
tw_error(TW_LOC, "Sending abort event!");
dest_node = tw_net_onnode((*e->src_lp->type->map)
((tw_lpid) e->dest_lp));
//if(!e->state.cancel_q)
//e->event_id = (tw_eventid) ++me->seq_num;
e->send_pe = (tw_peid) g_tw_mynode;
e->send_lp = e->src_lp->gid;
#if ROSS_MEMORY
// pack pointers
tmp_prev = e->prev;
tmp_lp = e->src_lp;
// delete when working
e->src_lp = NULL;
memory = NULL;
if(e->memory)
{
memory = e->memory;
e->memory = (tw_memory *) tw_memory_getsize(me, memory->fd);
e->prev = (tw_event *) memory->fd;
mem_size = (size_t) e->memory;
}
buffer = posted_sends.buffers[id];
memcpy(&buffer[position], e, g_tw_event_msg_sz);
position += g_tw_event_msg_sz;
// restore pointers
e->prev = tmp_prev;
e->src_lp = tmp_lp;
m = NULL;
while(memory)
{
m = memory->next;
if(m)
{
memory->next = (tw_memory *)
tw_memory_getsize(me, m->fd);
memory->fd = m->fd;
}
if(position + mem_size > TW_MEMORY_BUFFER_SIZE)
tw_error(TW_LOC, "Out of buffer space!");
memcpy(&buffer[position], memory, mem_size);
position += mem_size;
memory->nrefs--;
tw_memory_unshift(e->src_lp, memory, memory->fd);
if(NULL != (memory = m))
mem_size = tw_memory_getsize(me, memory->fd);
}
e->memory = NULL;
if (MPI_Isend(buffer,
EVENT_SIZE(e),
MPI_BYTE,
*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#else
if (MPI_Isend(e,
(int)EVENT_SIZE(e),
MPI_BYTE,
(int)*dest_node,
EVENT_TAG,
MPI_COMM_ROSS,
&posted_sends.req_list[id]) != MPI_SUCCESS) {
return changed;
}
#endif
tw_eventq_pop(&outq);
e->state.owner = e->state.cancel_q
? TW_net_acancel
: TW_net_asend;
posted_sends.event_list[id] = e;
posted_sends.cur++;
me->s_nwhite_sent++;
changed = 1;
}
return changed;
}
int
tw_socket_send_event(tw_event * event, tw_peid dest_peid)
{
tw_net_node *node;
tw_kp *kp;
tw_lp *src_lp;
//tw_lp *dest_lp;
#ifdef ROSS_MEMORY_LIB
tw_memory *memory;
tw_memory *m;
#endif
tw_event *temp_prev;
int rv;
int send_fd;
size_t mem_size;
mem_size = 0;
/*
* Save the lp pointers into temp vars and set lp pointers to lp->ids
*/
src_lp = event->src_lp;
//dest_lp = event->dest_lp;
temp_prev = event->prev;
kp = src_lp->kp;
event->src_lp = (tw_lp *) src_lp->gid;
// dest_lp contains LP global id
//event->dest_lp = (tw_lp *) dest_lp->id;
/*
* Store the sizeof the next memory buffer and the recv'r must figure
* that a memory buffer of that size is following the event.. then
* each successive memory buffer will also have the sizeof a
* following mem buffer until all are sent (memory->prev = 0)
*/
#ifdef ROSS_MEMORY_LIB
memory = NULL;
if(event->memory)
{
memory = event->memory;
event->memory = (tw_memory *)
tw_memory_getsize(kp, (tw_fd) memory->prev);
event->prev = (tw_event *) memory->prev;
mem_size = (size_t) event->memory;
}
#endif
node = g_tw_net_node[src_lp->pe->id];
send_fd = node->servers[dest_peid];
rv = tw_socket_send(send_fd, (char *) event, sizeof(tw_event) + g_tw_msg_sz, 100);
if (rv != sizeof(tw_event) + g_tw_msg_sz)
tw_error(TW_LOC, "Did not send full event! (%d of %d)",
rv, sizeof(tw_event) + g_tw_msg_sz);
/*
* Send the event's memory buffers
*/
#ifdef ROSS_MEMORY_LIB
m = NULL;
while(memory)
{
m = memory->next;
if(m)
{
memory->next = (tw_memory *)
tw_memory_getsize(kp, (tw_fd) m->prev);
memory->prev = (tw_memory *) m->prev;
}
rv = tw_socket_send(send_fd, (char *) memory, mem_size, 100);
#if VERIFY_SOCKET_TCP
printf("%d: sending mem buf of size %d on ev %f \n", src_lp->id, mem_size, event->recv_ts);
#endif
if(rv != mem_size)
tw_error(TW_LOC,
"Did not send full memory buffer! (%d of %d)",
rv, mem_size);
/* We will not need these mme bufs any longer on this side */
tw_memory_unshift(src_lp, memory, (tw_fd) memory->prev);
memory = m;
if(memory)
mem_size = tw_memory_getsize(src_lp, (tw_fd) memory->prev);
}
#endif
/*
* Restore the event's lp pointers
*/
event->src_lp = src_lp;
//event->dest_lp = dest_lp;
event->prev = temp_prev;
#ifdef ROSS_MEMORY_LIB
event->memory = NULL;
#endif
#if VERIFY_SOCKET_TCP
if (event->state.cancel_q)
printf
("SEND CANCEL: dest %d %d: ts=%f sn=%d src %d %d\n",
event->dest_lp->pe->id,
event->dest_lp->id, event->recv_ts,
event->seq_num,
event->src_lp->pe->id,
event->src_lp->id);
else
printf
("SEND NORMAL: dest %d %d: ts=%f sn=%d src %d %d\n",
event->dest_lp->pe->id,
event->dest_lp->id, event->recv_ts,
event->seq_num,
event->src_lp->pe->id,
event->src_lp->id);
#endif
return rv;
}
