/* return -1 on error */ static int psdapl_flush_sendbuf(psdapl_con_info_t *ci, char *lmem /* ci->send_bufs.lmr_mem */, unsigned roffset, unsigned size) { DAT_RETURN dat_rc; DAT_LMR_TRIPLET lmr; DAT_RMR_TRIPLET rmr; lmr.lmr_context = ci->send_bufs.lmr_context; lmr.pad = 0; lmr.virtual_address = psdapl_mem2vaddr(lmem); lmr.segment_length = size; rmr.rmr_context = ci->send_bufs.rmr_context; rmr.pad = 0; rmr.target_address = ci->send_bufs.rmr_vaddr + roffset; rmr.segment_length = size; DAT_DTO_COOKIE cookie; cookie.as_64 = 0; dat_rc = dat_ep_post_rdma_write(ci->ep_handle, 1, &lmr, cookie, &rmr, 0/* DAT_COMPLETION_SUPPRESS_FLAG*/); if (dat_rc != DAT_SUCCESS) goto err_rdma_write; return 0; err_rdma_write: psdapl_dprint_dat_err(0, dat_rc, "dat_ep_post_rdma_write() failed"); return -1; }
int mca_btl_udapl_put( mca_btl_base_module_t* btl, mca_btl_base_endpoint_t* endpoint, mca_btl_base_descriptor_t* des) { DAT_RMR_TRIPLET remote_buffer; DAT_DTO_COOKIE cookie; int rc = OMPI_SUCCESS; mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des; mca_btl_udapl_segment_t *dst_segment = des->des_dst; frag->btl = (mca_btl_udapl_module_t *)btl; frag->endpoint = endpoint; frag->type = MCA_BTL_UDAPL_PUT; if (OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) { /* no local work queue tokens available */ OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1); OPAL_THREAD_LOCK(&endpoint->endpoint_lock); opal_list_append(&endpoint->endpoint_max_frags, (opal_list_item_t*)frag); OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock); opal_progress(); } else { /* work queue tokens available, try to send */ if(OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) { OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1); OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], 1); OPAL_THREAD_LOCK(&endpoint->endpoint_lock); opal_list_append(&endpoint->endpoint_max_frags, (opal_list_item_t*)frag); OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock); opal_progress(); } else { frag->triplet.segment_length = frag->segment.base.seg_len; remote_buffer.rmr_context = dst_segment->context; remote_buffer.target_address = (DAT_VADDR)(uintptr_t)dst_segment->base.seg_addr.lval; remote_buffer.segment_length = dst_segment->base.seg_len; cookie.as_ptr = frag; OPAL_THREAD_LOCK(&endpoint->endpoint_lock); rc = dat_ep_post_rdma_write(endpoint->endpoint_max, 1, &frag->triplet, cookie, &remote_buffer, DAT_COMPLETION_DEFAULT_FLAG); OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock); if(DAT_SUCCESS != rc) { char* major; char* minor; dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_rdma_write", major, minor)); rc = OMPI_ERROR; } } } return rc; }
/* ----------------------------------------------------------- * Initiate an RDMA op (synchronous) on each of this thread's EPs. */ bool DT_handle_rdma_op(DT_Tdep_Print_Head * phead, Ep_Context_t * ep_context, unsigned int num_eps, DT_Transfer_Type opcode, int op_indx, bool poll) { unsigned int i, j; DAT_RETURN ret; unsigned char *completion_reaped; unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE]; bool rc = false; if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) { completion_reaped = lcomp; bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps); } else { completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char)); if (!completion_reaped) { return false; } } /* Initiate the operation */ for (i = 0; i < num_eps; i++) { Transaction_Test_Op_t *op = &ep_context[i].op[op_indx]; DAT_LMR_TRIPLET *iov = DT_Bpool_GetIOV(op->bp, 0); DAT_DTO_COOKIE cookie; DAT_RMR_TRIPLET rmr_triplet; /* Prep the inputs */ for (j = 0; j < op->num_segs; j++) { iov[j].virtual_address = (DAT_VADDR) (uintptr_t) DT_Bpool_GetBuffer(op->bp, j); iov[j].segment_length = op->seg_size; iov[j].lmr_context = DT_Bpool_GetLMR(op->bp, j); } cookie.as_64 = ((((DAT_UINT64) i) << 32) | (((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0)) & 0xffffffffUL)); rmr_triplet.virtual_address = (DAT_VADDR) (uintptr_t) op->Rdma_Address; rmr_triplet.segment_length = op->seg_size * op->num_segs; rmr_triplet.rmr_context = op->Rdma_Context; DT_Tdep_PT_Debug(3, (phead, "Call dat_ep_post_rdma_%s [" F64x ", sz=" F64x ", ctxt=%x]\n", (opcode == RDMA_WRITE ? "write" : "read"), rmr_triplet.virtual_address, rmr_triplet.segment_length, rmr_triplet.rmr_context)); /* Post the operation */ if (opcode == RDMA_WRITE) { ret = dat_ep_post_rdma_write(ep_context[i].ep_handle, op->num_segs, iov, cookie, &rmr_triplet, DAT_COMPLETION_DEFAULT_FLAG); } else { /* opcode == RDMA_READ */ ret = dat_ep_post_rdma_read(ep_context[i].ep_handle, op->num_segs, iov, cookie, &rmr_triplet, DAT_COMPLETION_DEFAULT_FLAG); } if (ret != DAT_SUCCESS) { DT_Tdep_PT_Printf(phead, "Test Error: dat_ep_post_rdma_%s failed: %s\n", (opcode == RDMA_WRITE ? "write" : "read"), DT_RetToString(ret)); DT_Test_Error(); goto err; } else { DT_Tdep_PT_Debug(3, (phead, "Done dat_ep_post_rdma_%s %s\n", (opcode == RDMA_WRITE ? "write" : "read"), " () Waiting ...")); } } /* Wait for it to happen */ for (i = 0; i < num_eps; i++) { Transaction_Test_Op_t *op; DAT_DTO_COMPLETION_EVENT_DATA dto_stat; DAT_DTO_COOKIE dto_cookie; unsigned int epnum; if (!DT_dto_event_reap (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) { goto err; } epnum = dto_stat.user_cookie.as_64 >> 32; if (epnum > num_eps) { DT_Tdep_PT_Printf(phead, "Test Error: %s: Invalid endpoint completion reaped.\n" "\tEndpoint: 0x%p, Cookie: 0x" F64x ", Length: " F64u "\n", opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", dto_stat.ep_handle, dto_stat.user_cookie.as_64, dto_stat.transfered_length); DT_Test_Error(); goto err; } op = &ep_context[epnum].op[op_indx]; dto_cookie.as_64 = ((((DAT_UINT64) epnum) << 32) | (((uintptr_t) DT_Bpool_GetBuffer(op->bp, 0)) & 0xffffffffUL)); if (!DT_dto_check(phead, &dto_stat, ep_context[epnum].ep_handle, op->num_segs * op->seg_size, dto_cookie, (opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))) { goto err; } if (completion_reaped[epnum]) { DT_Tdep_PT_Printf(phead, "Test Error: %s: Secondary completion seen for endpoint 0x%p (%d)\n", opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", ep_context[epnum].ep_handle, epnum); DT_Test_Error(); goto err; } completion_reaped[epnum] = 1; DT_Tdep_PT_Debug(3, (phead, "dat_ep_post_rdma_%s OK\n", (opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))); } for (i = 0; i < num_eps; i++) { if (completion_reaped[i] == 0) { DT_Tdep_PT_Printf(phead, "Test Error: %s: No completion seen for endpoint 0x%p (#%d)\n", opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", ep_context[i].ep_handle, i); DT_Test_Error(); goto err; } } rc = true; err: if (completion_reaped != lcomp) DT_Mdep_Free(completion_reaped); return rc; }