ucs_status_t ucp_put_nbi(ucp_ep_h ep, const void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
ucp_lane_index_t lane;
uct_rkey_t uct_rkey;
ucs_status_t status;
UCP_RMA_CHECK_PARAMS(buffer, length);
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config);
/* Fast path for a single short message */
if (length <= rma_config->max_put_short) {
status = uct_ep_put_short(ep->uct_eps[lane], buffer, length, remote_addr,
uct_rkey);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
/* Return on error or success */
return status;
}
}
ucp_rma_start_nbi(ep, buffer, length, remote_addr, rkey,
ucp_progress_put_nbi);
return UCS_INPROGRESS;
}
ucs_status_t ucp_get_nbi(ucp_ep_h ep, void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
ucp_lane_index_t lane;
uct_rkey_t uct_rkey;
ucs_status_t status;
UCP_RMA_CHECK_PARAMS(buffer, length);
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config);
if (length <= rma_config->max_get_bcopy) {
status = uct_ep_get_bcopy(ep->uct_eps[lane],
(uct_unpack_callback_t)memcpy,
(void*)buffer,
length,
remote_addr,
uct_rkey,
NULL);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
/* Return on error or success */
return status;
}
}
ucp_rma_start_nbi(ep, buffer, length, remote_addr, rkey,
ucp_progress_get_nbi);
return UCS_INPROGRESS;
}
ucs_status_t ucp_put(ucp_ep_h ep, const void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
size_t frag_length;
ssize_t packed_len;
ucp_lane_index_t lane;
UCP_RMA_CHECK_PARAMS(buffer, length);
/* Loop until all message has been sent.
* We re-check the configuration on every iteration, because it can be
* changed by transport switch.
*/
for (;;) {
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config);
if (length <= rma_config->max_put_short) {
status = uct_ep_put_short(ep->uct_eps[lane], buffer, length,
remote_addr, uct_rkey);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
break;
}
} else {
if (length <= ucp_ep_config(ep)->bcopy_thresh) {
frag_length = ucs_min(length, rma_config->max_put_short);
status = uct_ep_put_short(ep->uct_eps[lane], buffer, frag_length,
remote_addr, uct_rkey);
} else {
ucp_memcpy_pack_context_t pack_ctx;
pack_ctx.src = buffer;
pack_ctx.length = frag_length =
ucs_min(length, rma_config->max_put_bcopy);
packed_len = uct_ep_put_bcopy(ep->uct_eps[lane], ucp_memcpy_pack,
&pack_ctx, remote_addr, uct_rkey);
status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len;
}
if (ucs_likely(status == UCS_OK)) {
length -= frag_length;
if (length == 0) {
break;
}
buffer += frag_length;
remote_addr += frag_length;
} else if (status != UCS_ERR_NO_RESOURCE) {
break;
}
}
ucp_worker_progress(ep->worker);
}
return status;
}
ucs_status_t ucp_get(ucp_ep_h ep, void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
uct_completion_t comp;
ucs_status_t status;
uct_rkey_t uct_rkey;
size_t frag_length;
ucp_lane_index_t lane;
UCP_RMA_CHECK_PARAMS(buffer, length);
comp.count = 1;
for (;;) {
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, lane, uct_rkey, rma_config);
/* Push out all fragments, and request completion only for the last
* fragment.
*/
frag_length = ucs_min(rma_config->max_get_bcopy, length);
status = uct_ep_get_bcopy(ep->uct_eps[lane], (uct_unpack_callback_t)memcpy,
(void*)buffer, frag_length, remote_addr,
uct_rkey, &comp);
if (ucs_likely(status == UCS_OK)) {
goto posted;
} else if (status == UCS_INPROGRESS) {
++comp.count;
goto posted;
} else if (status == UCS_ERR_NO_RESOURCE) {
goto retry;
} else {
return status;
}
posted:
length -= frag_length;
if (length == 0) {
break;
}
buffer += frag_length;
remote_addr += frag_length;
retry:
ucp_worker_progress(ep->worker);
}
/* coverity[loop_condition] */
while (comp.count > 1) {
ucp_worker_progress(ep->worker);
}
return UCS_OK;
}
static ucs_status_t ucp_progress_put_nbi(uct_pending_req_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_rkey_h rkey = req->send.rma.rkey;
ucp_ep_t *ep = req->send.ep;
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
ssize_t packed_len;
uct_ep_h uct_ep;
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config);
for (;;) {
if (req->send.length <= ep->worker->context->config.ext.bcopy_thresh) {
/* Should be replaced with bcopy */
packed_len = ucs_min(req->send.length, rma_config->max_put_short);
status = uct_ep_put_short(uct_ep,
req->send.buffer,
packed_len,
req->send.rma.remote_addr,
uct_rkey);
} else {
/* We don't do it right now, but in future we have to add
* an option to use zcopy
*/
ucp_memcpy_pack_context_t pack_ctx;
pack_ctx.src = req->send.buffer;
pack_ctx.length =
ucs_min(req->send.length, rma_config->max_put_bcopy);
packed_len = uct_ep_put_bcopy(uct_ep,
ucp_memcpy_pack,
&pack_ctx,
req->send.rma.remote_addr,
uct_rkey);
status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len;
}
if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) {
req->send.length -= packed_len;
if (req->send.length == 0) {
ucp_request_complete(req, void);
break;
}
req->send.buffer += packed_len;
req->send.rma.remote_addr += packed_len;
} else {
break;
}
}
return status;
}
ucs_status_t ucp_get_nbi(ucp_ep_h ep, void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
size_t frag_length;
uct_ep_h uct_ep;
UCP_RMA_CHECK_PARAMS(buffer, length);
for (;;) {
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config);
frag_length = ucs_min(rma_config->max_get_bcopy, length);
status = uct_ep_get_bcopy(uct_ep,
(uct_unpack_callback_t)memcpy,
(void*)buffer,
frag_length,
remote_addr,
uct_rkey,
NULL);
if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) {
/* Get was initiated */
length -= frag_length;
buffer += frag_length;
remote_addr += frag_length;
if (length == 0) {
break;
}
} else if (ucs_unlikely(status == UCS_ERR_NO_RESOURCE)) {
/* Out of resources - adding request for later schedule */
ucp_request_t *req;
req = ucs_mpool_get_inline(&ep->worker->req_mp);
if (req == NULL) {
/* can't allocate memory for request - abort */
status = UCS_ERR_NO_MEMORY;
break;
}
ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr,
rkey, ucp_progress_get_nbi);
/* Mark it as in progress */
status = UCS_INPROGRESS;
break;
} else {
/* Error */
break;
}
}
return status;
}
static ucs_status_t ucp_progress_get_nbi(uct_pending_req_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_rkey_h rkey = req->send.rma.rkey;
ucp_ep_t *ep = req->send.ep;
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
size_t frag_length;
uct_ep_h uct_ep;
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config);
for (;;) {
frag_length = ucs_min(rma_config->max_get_bcopy, req->send.length);
status = uct_ep_get_bcopy(uct_ep,
(uct_unpack_callback_t)memcpy,
(void*)req->send.buffer,
frag_length,
req->send.rma.remote_addr,
uct_rkey,
NULL);
if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) {
/* Get was initiated */
req->send.length -= frag_length;
req->send.buffer += frag_length;
req->send.rma.remote_addr += frag_length;
if (req->send.length == 0) {
/* Get was posted */
ucp_request_complete(req, void);
status = UCS_OK;
break;
}
} else {
/* Error - abort */
break;
}
}
return status;
}
static ucs_status_t ucp_progress_get_nbi(uct_pending_req_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_rkey_h rkey = req->send.rma.rkey;
ucp_ep_t *ep = req->send.ep;
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
size_t frag_length;
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, req->send.lane, uct_rkey, rma_config);
frag_length = ucs_min(rma_config->max_get_bcopy, req->send.length);
status = uct_ep_get_bcopy(ep->uct_eps[req->send.lane],
(uct_unpack_callback_t)memcpy,
(void*)req->send.buffer,
frag_length,
req->send.rma.remote_addr,
uct_rkey,
NULL);
if ((status == UCS_OK) || (status == UCS_INPROGRESS)) {
/* Get was initiated */
req->send.length -= frag_length;
req->send.buffer += frag_length;
req->send.rma.remote_addr += frag_length;
if (req->send.length == 0) {
/* Get was posted */
ucp_request_put(req, UCS_OK);
return UCS_OK;
} else {
return UCS_INPROGRESS;
}
} else {
return status;
}
}
ucs_status_t ucp_put_nbi(ucp_ep_h ep, const void *buffer, size_t length,
uint64_t remote_addr, ucp_rkey_h rkey)
{
ucp_ep_rma_config_t *rma_config;
ucs_status_t status;
uct_rkey_t uct_rkey;
ssize_t packed_len;
ucp_request_t *req;
uct_ep_h uct_ep;
UCP_RMA_CHECK_PARAMS(buffer, length);
for (;;) {
UCP_EP_RESOLVE_RKEY_RMA(ep, rkey, uct_ep, uct_rkey, rma_config);
if (length <= rma_config->max_put_short) {
/* Fast path for a single short message */
status = uct_ep_put_short(uct_ep, buffer, length, remote_addr,
uct_rkey);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
/* Return on error or success */
break;
} else {
/* Out of resources - adding request for later schedule */
req = ucs_mpool_get_inline(&ep->worker->req_mp);
if (req == NULL) {
status = UCS_ERR_NO_MEMORY;
break;
}
ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr,
rkey, ucp_progress_put_nbi);
status = UCS_INPROGRESS;
break;
}
} else {
/* Fragmented put */
if (length <= ucp_ep_config(ep)->bcopy_thresh) {
/* TBD: Should be replaced with bcopy */
packed_len = ucs_min(length, rma_config->max_put_short);
status = uct_ep_put_short(uct_ep, buffer, packed_len, remote_addr,
uct_rkey);
} else {
/* TBD: Use z-copy */
ucp_memcpy_pack_context_t pack_ctx;
pack_ctx.src = buffer;
pack_ctx.length =
ucs_min(length, rma_config->max_put_bcopy);
packed_len = uct_ep_put_bcopy(uct_ep, ucp_memcpy_pack, &pack_ctx,
remote_addr, uct_rkey);
status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len;
}
if (ucs_likely(status == UCS_OK || status == UCS_INPROGRESS)) {
length -= packed_len;
if (length == 0) {
/* Put is completed - return success */
break;
}
buffer += packed_len;
remote_addr += packed_len;
} else if (status == UCS_ERR_NO_RESOURCE) {
/* Out of resources - adding request for later schedule */
req = ucs_mpool_get_inline(&ep->worker->req_mp);
if (req == NULL) {
status = UCS_ERR_NO_MEMORY;
break;
}
ucp_add_pending_rma(req, ep, uct_ep, buffer, length, remote_addr,
rkey, ucp_progress_put_nbi);
status = UCS_INPROGRESS;
break;
} else {
/* Return - Error occured */
break;
}
}
}
return status;
}
