void
MPIDI_CH3I_MRAILI_Rendezvous_rput_push (MPIDI_VC_t * vc, MPID_Request * sreq)
{
vbuf *v;
int i = 0;
MRAILI_Channel_info channel;
channel.hca_index = 0;
channel.rail_index = 0;
int nbytes;
if (sreq->mrail.rndv_buf_off != 0)
{
udapl_error_abort (GEN_ASSERT_ERR,
"s->bytes_sent != 0 Rendezvous Push, %d",
sreq->mrail.nearly_complete);
}
#if defined(DEBUG)
if (sreq->mrail.rndv_buf_sz > 0)
{
MPIU_Assert (sreq->mrail.d_entry != NULL);
MPIU_Assert (sreq->mrail.remote_addr != NULL);
}
#endif /* defined(DEBUG) */
while (sreq->mrail.rndv_buf_off < sreq->mrail.rndv_buf_sz)
{
v = get_vbuf ();
nbytes = sreq->mrail.rndv_buf_sz - sreq->mrail.rndv_buf_off;
if (nbytes > MPIDI_CH3I_RDMA_Process.maxtransfersize)
{
nbytes = MPIDI_CH3I_RDMA_Process.maxtransfersize;
}
DEBUG_PRINT("[buffer content]: %02x,%02x,%02x, offset %d, remote buf %p\n",
((char *) sreq->mrail.rndv_buf)[0],
((char *) sreq->mrail.rndv_buf)[1],
((char *) sreq->mrail.rndv_buf)[2],
sreq->mrail.rndv_buf_off, sreq->mrail.remote_addr);
MRAILI_RDMA_Put (vc, v,
(char *) (sreq->mrail.rndv_buf) +
sreq->mrail.rndv_buf_off,
((dreg_entry *) sreq->mrail.d_entry)->memhandle,
(char *) (sreq->mrail.remote_addr) +
sreq->mrail.rndv_buf_off,
sreq->mrail.remote_handle, nbytes, &channel);
sreq->mrail.rndv_buf_off += nbytes;
}
MPIU_Assert (sreq->mrail.rndv_buf_off == sreq->mrail.rndv_buf_sz);
for (; i < vc->mrail.num_total_subrails; ++i)
{
/*Fix ME: only support one channel */
channel.rail_index = i;
MRAILI_RDMA_Put_finish (vc, sreq, &channel);
break;
}
sreq->mrail.nearly_complete = 1;
}
void vbuf_rdma_address_send (viadev_connection_t * c)
{
/* inform others about the receive
* RDMA buffer address and handle */
vbuf *v;
viadev_packet_rdma_address *h;
v = get_vbuf();
h = (viadev_packet_rdma_address *) VBUF_BUFFER_START(v);
/* set up the packet */
PACKET_SET_HEADER(h, c, VIADEV_RDMA_ADDRESS);
h->envelope.src_lrank = c->global_rank;
/* set address and handle */
h->RDMA_address = c->RDMA_recv_buf_DMA;
h->RDMA_hndl = c->RDMA_recv_buf_hndl->rkey;
#ifdef MCST_SUPPORT
if (is_mcast_enabled) {
/* set address and handle for UD Acks, this is for now remove later */
h->RDMA_ACK_address = (viadev.bcast_info.ack_buffer);
h->RDMA_ACK_hndl.rkey = viadev.bcast_info.ack_mem_hndl.rkey;
}
#endif /* MCST_SUPPORT */
/* finally send it off */
vbuf_init_send(v, sizeof(viadev_packet_rdma_address));
viadev_post_send(c, v);
}
const char* session::get(const char* name, bool local_cached /* = false */)
{
const VBUF* bf = get_vbuf(name, local_cached);
if (bf == NULL)
return "";
return bf->buf;
}
int MPIDI_CH3I_MRAILI_Get_rndv_rput(MPIDI_VC_t *vc,
MPID_Request * req,
MPIDI_CH3I_MRAILI_Rndv_info_t * rndv,
MPID_IOV *iov)
{
/* This function will register the local buf, send rdma write to target, and send
* get_resp_kt as rput finsh. Currently, we assume the local buffer is contiguous,
* datatype cases will be considered later */
int nbytes;
int rail;
vbuf *v;
MPIDI_STATE_DECL(MPIDI_STATE_GEN2_RNDV_RPUT);
MPIDI_FUNC_ENTER(MPIDI_STATE_GEN2_RNDV_RPUT);
MPIDI_CH3I_MRAIL_Prepare_rndv(vc, req);
MPIDI_CH3I_MRAIL_REVERT_RPUT(req);
if (VAPI_PROTOCOL_RPUT == req->mrail.protocol) {
MPIDI_CH3I_MRAIL_Prepare_rndv_transfer(req, rndv);
}
rail = MRAILI_Send_select_rail(vc);
/* STEP 2: Push RDMA write */
while ((req->mrail.rndv_buf_off < req->mrail.rndv_buf_sz)
&& VAPI_PROTOCOL_RPUT == req->mrail.protocol) {
v = get_vbuf();
v->sreq = req;
MPIU_Assert(v != NULL);
nbytes = req->mrail.rndv_buf_sz - req->mrail.rndv_buf_off;
if (nbytes > MPIDI_CH3I_RDMA_Process.maxtransfersize) {
nbytes = MPIDI_CH3I_RDMA_Process.maxtransfersize;
}
DEBUG_PRINT("[buffer content]: offset %d\n", req->mrail.rndv_buf_off);
MRAILI_RDMA_Put(vc, v,
(char *) (req->mrail.rndv_buf) + req->mrail.rndv_buf_off,
((dreg_entry *) req->mrail.d_entry)->memhandle[vc->
mrail.rails[rail].hca_index]->lkey,
(char *) (req->mrail.remote_addr) +
req->mrail.rndv_buf_off,
req->mrail.rkey[vc->mrail.rails[rail].hca_index],
nbytes, rail);
req->mrail.rndv_buf_off += nbytes;
}
if (VAPI_PROTOCOL_RPUT == req->mrail.protocol) {
MPIDI_CH3I_MRAILI_rput_complete(vc, iov, 1, &nbytes, &v, rail);
v->sreq = req;
}
MPIDI_FUNC_EXIT(MPIDI_STATE_GEN2_RNDV_RPUT);
return MPI_SUCCESS;
}
static inline vbuf * get_eager_coalesce_vbuf(viadev_connection_t * c,
int len, viadev_packet_envelope * envelope) {
vbuf * tail_v;
vbuf * to_use = NULL;
int total_len =
len + sizeof(viadev_packet_eager_coalesce_full) +
VBUF_FAST_RDMA_EXTRA_BYTES;
/* if we already have an entry, does it have room? */
if(c->ext_sendq_head) {
tail_v = c->ext_sendq_tail;
/* is it of the proper type? */
if(tail_v->len != 0 && tail_v->bytes_remaining >= total_len &&
(!viadev_use_eager_coalesce_limited ||
match_cached_out_envelope(c, envelope))) {
#ifdef DEBUG
viadev_packet_header * tail_h =
(viadev_packet_header *) tail_v->buffer;
assert(tail_h->type == VIADEV_PACKET_EAGER_COALESCE);
#endif
return tail_v;
} else {
/* there are items in the sendq that we cannot
* coalesce to anymore -- so just send it out
*/
viadev_ext_sendq_send(c);
}
}
D_PRINT("No luck with looking for existing ones\n");
/* if we get here we need to start a new packet */
{
viadev_packet_eager_coalesce * h;
total_len += sizeof(viadev_packet_eager_coalesce);
/* try to get an FP one */
#ifdef ADAPTIVE_RDMA_FAST_PATH
{
int rdma_ok;
rdma_ok = fast_rdma_ok(c, total_len, 1);
if(rdma_ok) {
to_use = &(c->RDMA_send_buf[c->phead_RDMA_send]);
c->RDMA_send_buf[c->phead_RDMA_send].padding = BUSY_FLAG;
if (++(c->phead_RDMA_send) >= viadev_num_rdma_buffer)
c->phead_RDMA_send = 0;
h = (viadev_packet_eager_coalesce *) to_use->buffer;
to_use->bytes_remaining = VBUF_BUFFER_SIZE -
VBUF_FAST_RDMA_EXTRA_BYTES;
}
}
#endif
if(NULL == to_use) {
D_PRINT("Getting a vbuf for send/recv\n");
/* take one for send/recv */
to_use = get_vbuf();
to_use->bytes_remaining = VBUF_BUFFER_SIZE;
h = (viadev_packet_eager_coalesce *) VBUF_BUFFER_START(to_use);
D_PRINT("buf: %p, h: %p\n", to_use->buffer, h);
if(!viadev_use_srq) {
c->remote_credit--;
}
}
/* few bookkeeping items are needed to start a coalesced
* packet
*/
h->pkt_count = 0;
h->header.id = c->next_packet_tosend++;
h->header.type = VIADEV_PACKET_EAGER_COALESCE;
h->header.src_rank = viadev.me;
h->header.vbuf_credit = 0;
h->header.remote_credit = 0;
#ifdef ADAPTIVE_RDMA_FAST_PATH
h->header.rdma_credit = 0;
#endif
h->header.ack = 0;
/* need to add this to the queue */
viadev_ext_sendq_queue(c, to_use);
to_use->shandle = NULL;
to_use->data_start = ((unsigned char *) h) + sizeof(*h);
to_use->grank = c->global_rank;
to_use->len = sizeof(*h);
to_use->bytes_remaining -= to_use->len;
}
return to_use;
}
