ucs_status_t ucp_put_nbi(ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; ucp_lane_index_t lane; uct_rkey_t uct_rkey; ucs_status_t status; UCP_RMA_CHECK_PARAMS(buffer, length); UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config); /* Fast path for a single short message */ if (length <= rma_config->max_put_short) { status = uct_ep_put_short(ep->uct_eps[lane], buffer, length, remote_addr, uct_rkey); if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) { /* Return on error or success */ return status; } } ucp_rma_start_nbi(ep, buffer, length, remote_addr, rkey, ucp_progress_put_nbi); return UCS_INPROGRESS; }
ucs_status_t ucp_get_nbi(ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; ucp_lane_index_t lane; uct_rkey_t uct_rkey; ucs_status_t status; UCP_RMA_CHECK_PARAMS(buffer, length); UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config); if (length <= rma_config->max_get_bcopy) { status = uct_ep_get_bcopy(ep->uct_eps[lane], (uct_unpack_callback_t)memcpy, (void*)buffer, length, remote_addr, uct_rkey, NULL); if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) { /* Return on error or success */ return status; } } ucp_rma_start_nbi(ep, buffer, length, remote_addr, rkey, ucp_progress_get_nbi); return UCS_INPROGRESS; }
ucs_status_t ucp_put(ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; size_t frag_length; ssize_t packed_len; ucp_lane_index_t lane; UCP_RMA_CHECK_PARAMS(buffer, length); /* Loop until all message has been sent. * We re-check the configuration on every iteration, because it can be * changed by transport switch. */ for (;;) { UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config); if (length <= rma_config->max_put_short) { status = uct_ep_put_short(ep->uct_eps[lane], buffer, length, remote_addr, uct_rkey); if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) { break; } } else { if (length <= ucp_ep_config(ep)->bcopy_thresh) { frag_length = ucs_min(length, rma_config->max_put_short); status = uct_ep_put_short(ep->uct_eps[lane], buffer, frag_length, remote_addr, uct_rkey); } else { ucp_memcpy_pack_context_t pack_ctx; pack_ctx.src = buffer; pack_ctx.length = frag_length = ucs_min(length, rma_config->max_put_bcopy); packed_len = uct_ep_put_bcopy(ep->uct_eps[lane], ucp_memcpy_pack, &pack_ctx, remote_addr, uct_rkey); status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len; } if (ucs_likely(status == UCS_OK)) { length -= frag_length; if (length == 0) { break; } buffer += frag_length; remote_addr += frag_length; } else if (status != UCS_ERR_NO_RESOURCE) { break; } } ucp_worker_progress(ep->worker); } return status; }
ucs_status_t ucp_get(ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; uct_completion_t comp; ucs_status_t status; uct_rkey_t uct_rkey; size_t frag_length; ucp_lane_index_t lane; UCP_RMA_CHECK_PARAMS(buffer, length); comp.count = 1; for (;;) { UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config); /* Push out all fragments, and request completion only for the last * fragment. */ frag_length = ucs_min(rma_config->max_get_bcopy, length); status = uct_ep_get_bcopy(ep->uct_eps[lane], (uct_unpack_callback_t)memcpy, (void*)buffer, frag_length, remote_addr, uct_rkey, &comp); if (ucs_likely(status == UCS_OK)) { goto posted; } else if (status == UCS_INPROGRESS) { ++comp.count; goto posted; } else if (status == UCS_ERR_NO_RESOURCE) { goto retry; } else { return status; } posted: length -= frag_length; if (length == 0) { break; } buffer += frag_length; remote_addr += frag_length; retry: ucp_worker_progress(ep->worker); } /* coverity[loop_condition] */ while (comp.count > 1) { ucp_worker_progress(ep->worker); } return UCS_OK; }
static ucs_status_t ucp_progress_put_nbi(uct_pending_req_t *self) { ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct); ucp_rkey_h rkey = req->send.rma.rkey; ucp_ep_t *ep = req->send.ep; ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; ssize_t packed_len; uct_ep_h uct_ep; UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config); for (;;) { if (req->send.length <= ep->worker->context->config.ext.bcopy_thresh) { /* Should be replaced with bcopy */ packed_len = ucs_min(req->send.length, rma_config->max_put_short); status = uct_ep_put_short(uct_ep, req->send.buffer, packed_len, req->send.rma.remote_addr, uct_rkey); } else { /* We don't do it right now, but in future we have to add * an option to use zcopy */ ucp_memcpy_pack_context_t pack_ctx; pack_ctx.src = req->send.buffer; pack_ctx.length = ucs_min(req->send.length, rma_config->max_put_bcopy); packed_len = uct_ep_put_bcopy(uct_ep, ucp_memcpy_pack, &pack_ctx, req->send.rma.remote_addr, uct_rkey); status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len; } if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) { req->send.length -= packed_len; if (req->send.length == 0) { ucp_request_complete(req, void); break; } req->send.buffer += packed_len; req->send.rma.remote_addr += packed_len; } else { break; } } return status; }
ucs_status_t ucp_get_nbi(ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; size_t frag_length; uct_ep_h uct_ep; UCP_RMA_CHECK_PARAMS(buffer, length); for (;;) { UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config); frag_length = ucs_min(rma_config->max_get_bcopy, length); status = uct_ep_get_bcopy(uct_ep, (uct_unpack_callback_t)memcpy, (void*)buffer, frag_length, remote_addr, uct_rkey, NULL); if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) { /* Get was initiated */ length -= frag_length; buffer += frag_length; remote_addr += frag_length; if (length == 0) { break; } } else if (ucs_unlikely(status == UCS_ERR_NO_RESOURCE)) { /* Out of resources - adding request for later schedule */ ucp_request_t *req; req = ucs_mpool_get_inline(&ep->worker->req_mp); if (req == NULL) { /* can't allocate memory for request - abort */ status = UCS_ERR_NO_MEMORY; break; } ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr, rkey, ucp_progress_get_nbi); /* Mark it as in progress */ status = UCS_INPROGRESS; break; } else { /* Error */ break; } } return status; }
static ucs_status_t ucp_progress_get_nbi(uct_pending_req_t *self) { ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct); ucp_rkey_h rkey = req->send.rma.rkey; ucp_ep_t *ep = req->send.ep; ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; size_t frag_length; uct_ep_h uct_ep; UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config); for (;;) { frag_length = ucs_min(rma_config->max_get_bcopy, req->send.length); status = uct_ep_get_bcopy(uct_ep, (uct_unpack_callback_t)memcpy, (void*)req->send.buffer, frag_length, req->send.rma.remote_addr, uct_rkey, NULL); if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) { /* Get was initiated */ req->send.length -= frag_length; req->send.buffer += frag_length; req->send.rma.remote_addr += frag_length; if (req->send.length == 0) { /* Get was posted */ ucp_request_complete(req, void); status = UCS_OK; break; } } else { /* Error - abort */ break; } } return status; }
static ucs_status_t ucp_progress_get_nbi(uct_pending_req_t *self) { ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct); ucp_rkey_h rkey = req->send.rma.rkey; ucp_ep_t *ep = req->send.ep; ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; size_t frag_length; UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, req->send.lane, uct_rkey, rma_config); frag_length = ucs_min(rma_config->max_get_bcopy, req->send.length); status = uct_ep_get_bcopy(ep->uct_eps[req->send.lane], (uct_unpack_callback_t)memcpy, (void*)req->send.buffer, frag_length, req->send.rma.remote_addr, uct_rkey, NULL); if ((status == UCS_OK) || (status == UCS_INPROGRESS)) { /* Get was initiated */ req->send.length -= frag_length; req->send.buffer += frag_length; req->send.rma.remote_addr += frag_length; if (req->send.length == 0) { /* Get was posted */ ucp_request_put(req, UCS_OK); return UCS_OK; } else { return UCS_INPROGRESS; } } else { return status; } }
ucs_status_t ucp_put_nbi(ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey) { ucp_ep_rma_config_t *rma_config; ucs_status_t status; uct_rkey_t uct_rkey; ssize_t packed_len; ucp_request_t *req; uct_ep_h uct_ep; UCP_RMA_CHECK_PARAMS(buffer, length); for (;;) { UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config); if (length <= rma_config->max_put_short) { /* Fast path for a single short message */ status = uct_ep_put_short(uct_ep, buffer, length, remote_addr, uct_rkey); if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) { /* Return on error or success */ break; } else { /* Out of resources - adding request for later schedule */ req = ucs_mpool_get_inline(&ep->worker->req_mp); if (req == NULL) { status = UCS_ERR_NO_MEMORY; break; } ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr, rkey, ucp_progress_put_nbi); status = UCS_INPROGRESS; break; } } else { /* Fragmented put */ if (length <= ucp_ep_config(ep)->bcopy_thresh) { /* TBD: Should be replaced with bcopy */ packed_len = ucs_min(length, rma_config->max_put_short); status = uct_ep_put_short(uct_ep, buffer, packed_len, remote_addr, uct_rkey); } else { /* TBD: Use z-copy */ ucp_memcpy_pack_context_t pack_ctx; pack_ctx.src = buffer; pack_ctx.length = ucs_min(length, rma_config->max_put_bcopy); packed_len = uct_ep_put_bcopy(uct_ep, ucp_memcpy_pack, &pack_ctx, remote_addr, uct_rkey); status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len; } if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) { length -= packed_len; if (length == 0) { /* Put is completed - return success */ break; } buffer += packed_len; remote_addr += packed_len; } else if (status == UCS_ERR_NO_RESOURCE) { /* Out of resources - adding request for later schedule */ req = ucs_mpool_get_inline(&ep->worker->req_mp); if (req == NULL) { status = UCS_ERR_NO_MEMORY; break; } ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr, rkey, ucp_progress_put_nbi); status = UCS_INPROGRESS; break; } else { /* Return - Error occured */ break; } } } return status; }