/* return -1 on error */
static
int psdapl_flush_sendbuf(psdapl_con_info_t *ci,
char *lmem /* ci->send_bufs.lmr_mem */,
unsigned roffset, unsigned size)
{
DAT_RETURN dat_rc;
DAT_LMR_TRIPLET lmr;
DAT_RMR_TRIPLET rmr;
lmr.lmr_context = ci->send_bufs.lmr_context;
lmr.pad = 0;
lmr.virtual_address = psdapl_mem2vaddr(lmem);
lmr.segment_length = size;
rmr.rmr_context = ci->send_bufs.rmr_context;
rmr.pad = 0;
rmr.target_address = ci->send_bufs.rmr_vaddr + roffset;
rmr.segment_length = size;
DAT_DTO_COOKIE cookie;
cookie.as_64 = 0;
dat_rc = dat_ep_post_rdma_write(ci->ep_handle, 1,
&lmr, cookie, &rmr, 0/* DAT_COMPLETION_SUPPRESS_FLAG*/);
if (dat_rc != DAT_SUCCESS) goto err_rdma_write;
return 0;
err_rdma_write:
psdapl_dprint_dat_err(0, dat_rc, "dat_ep_post_rdma_write() failed");
return -1;
}
int mca_btl_udapl_put(
mca_btl_base_module_t* btl,
mca_btl_base_endpoint_t* endpoint,
mca_btl_base_descriptor_t* des)
{
DAT_RMR_TRIPLET remote_buffer;
DAT_DTO_COOKIE cookie;
int rc = OMPI_SUCCESS;
mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
mca_btl_udapl_segment_t *dst_segment = des->des_dst;
frag->btl = (mca_btl_udapl_module_t *)btl;
frag->endpoint = endpoint;
frag->type = MCA_BTL_UDAPL_PUT;
if (OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
/* no local work queue tokens available */
OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
opal_list_append(&endpoint->endpoint_max_frags,
(opal_list_item_t*)frag);
OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
opal_progress();
} else {
/* work queue tokens available, try to send */
if(OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
opal_list_append(&endpoint->endpoint_max_frags,
(opal_list_item_t*)frag);
OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
opal_progress();
} else {
frag->triplet.segment_length = frag->segment.base.seg_len;
remote_buffer.rmr_context = dst_segment->context;
remote_buffer.target_address =
(DAT_VADDR)(uintptr_t)dst_segment->base.seg_addr.lval;
remote_buffer.segment_length = dst_segment->base.seg_len;
cookie.as_ptr = frag;
OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
rc = dat_ep_post_rdma_write(endpoint->endpoint_max,
1,
&frag->triplet,
cookie,
&remote_buffer,
DAT_COMPLETION_DEFAULT_FLAG);
OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
if(DAT_SUCCESS != rc) {
char* major;
char* minor;
dat_strerror(rc, (const char**)&major,
(const char**)&minor);
BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_rdma_write",
major, minor));
rc = OMPI_ERROR;
}
}
}
return rc;
}
/* -----------------------------------------------------------
* Initiate an RDMA op (synchronous) on each of this thread's EPs.
*/
bool
DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
Ep_Context_t * ep_context,
unsigned int num_eps,
DT_Transfer_Type opcode, int op_indx, bool poll)
{
unsigned int i, j;
DAT_RETURN ret;
unsigned char *completion_reaped;
unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
bool rc = false;
if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) {
completion_reaped = lcomp;
bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps);
}
else {
completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
if (!completion_reaped) {
return false;
}
}
/* Initiate the operation */
for (i = 0; i < num_eps; i++) {
Transaction_Test_Op_t *op = &ep_context[i].op[op_indx];
DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV(op->bp, 0);
DAT_DTO_COOKIE cookie;
DAT_RMR_TRIPLET rmr_triplet;
/* Prep the inputs */
for (j = 0; j < op->num_segs; j++) {
iov[j].virtual_address = (DAT_VADDR) (uintptr_t)
DT_Bpool_GetBuffer(op->bp, j);
iov[j].segment_length = op->seg_size;
iov[j].lmr_context = DT_Bpool_GetLMR(op->bp, j);
}
cookie.as_64 = ((((DAT_UINT64) i) << 32)
| (((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0)) &
0xffffffffUL));
rmr_triplet.virtual_address =
(DAT_VADDR) (uintptr_t) op->Rdma_Address;
rmr_triplet.segment_length = op->seg_size * op->num_segs;
rmr_triplet.rmr_context = op->Rdma_Context;
DT_Tdep_PT_Debug(3, (phead,
"Call dat_ep_post_rdma_%s [" F64x ", sz="
F64x ", ctxt=%x]\n",
(opcode == RDMA_WRITE ? "write" : "read"),
rmr_triplet.virtual_address,
rmr_triplet.segment_length,
rmr_triplet.rmr_context));
/* Post the operation */
if (opcode == RDMA_WRITE) {
ret = dat_ep_post_rdma_write(ep_context[i].ep_handle,
op->num_segs,
iov,
cookie,
&rmr_triplet,
DAT_COMPLETION_DEFAULT_FLAG);
} else { /* opcode == RDMA_READ */
ret = dat_ep_post_rdma_read(ep_context[i].ep_handle,
op->num_segs,
iov,
cookie,
&rmr_triplet,
DAT_COMPLETION_DEFAULT_FLAG);
}
if (ret != DAT_SUCCESS) {
DT_Tdep_PT_Printf(phead,
"Test Error: dat_ep_post_rdma_%s failed: %s\n",
(opcode ==
RDMA_WRITE ? "write" : "read"),
DT_RetToString(ret));
DT_Test_Error();
goto err;
} else {
DT_Tdep_PT_Debug(3, (phead,
"Done dat_ep_post_rdma_%s %s\n",
(opcode ==
RDMA_WRITE ? "write" : "read"),
" () Waiting ..."));
}
}
/* Wait for it to happen */
for (i = 0; i < num_eps; i++) {
Transaction_Test_Op_t *op;
DAT_DTO_COMPLETION_EVENT_DATA dto_stat;
DAT_DTO_COOKIE dto_cookie;
unsigned int epnum;
if (!DT_dto_event_reap
(phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) {
goto err;
}
epnum = dto_stat.user_cookie.as_64 >> 32;
if (epnum > num_eps) {
DT_Tdep_PT_Printf(phead,
"Test Error: %s: Invalid endpoint completion reaped.\n"
"\tEndpoint: 0x%p, Cookie: 0x" F64x
", Length: " F64u "\n",
opcode ==
RDMA_WRITE ? "RDMA/WR" : "RDMA/RD",
dto_stat.ep_handle,
dto_stat.user_cookie.as_64,
dto_stat.transfered_length);
DT_Test_Error();
goto err;
}
op = &ep_context[epnum].op[op_indx];
dto_cookie.as_64 = ((((DAT_UINT64) epnum) << 32)
|
(((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0))
& 0xffffffffUL));
if (!DT_dto_check(phead,
&dto_stat,
ep_context[epnum].ep_handle,
op->num_segs * op->seg_size,
dto_cookie,
(opcode ==
RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))) {
goto err;
}
if (completion_reaped[epnum]) {
DT_Tdep_PT_Printf(phead,
"Test Error: %s: Secondary completion seen for endpoint 0x%p (%d)\n",
opcode ==
RDMA_WRITE ? "RDMA/WR" : "RDMA/RD",
ep_context[epnum].ep_handle, epnum);
DT_Test_Error();
goto err;
}
completion_reaped[epnum] = 1;
DT_Tdep_PT_Debug(3, (phead,
"dat_ep_post_rdma_%s OK\n",
(opcode ==
RDMA_WRITE ? "RDMA/WR" : "RDMA/RD")));
}
for (i = 0; i < num_eps; i++) {
if (completion_reaped[i] == 0) {
DT_Tdep_PT_Printf(phead,
"Test Error: %s: No completion seen for endpoint 0x%p (#%d)\n",
opcode ==
RDMA_WRITE ? "RDMA/WR" : "RDMA/RD",
ep_context[i].ep_handle, i);
DT_Test_Error();
goto err;
}
}
rc = true;
err:
if (completion_reaped != lcomp)
DT_Mdep_Free(completion_reaped);
return rc;
}