static UCS_F_ALWAYS_INLINE ucs_status_ptr_t
ucp_tag_send_req(ucp_request_t *req, size_t count,
const ucp_ep_msg_config_t* msg_config,
size_t rndv_rma_thresh, size_t rndv_am_thresh,
ucp_send_callback_t cb, const ucp_proto_t *proto)
{
size_t seg_size = (msg_config->max_bcopy - proto->only_hdr_size);
size_t rndv_thresh = ucp_tag_get_rndv_threshold(req, count,
msg_config->max_iov,
rndv_rma_thresh,
rndv_am_thresh, seg_size);
size_t zcopy_thresh = ucp_proto_get_zcopy_threshold(req, msg_config, count,
rndv_thresh);
ssize_t max_short = ucp_proto_get_short_max(req, msg_config);
ucs_status_t status;
ucs_trace_req("select tag request(%p) progress algorithm datatype=%lx "
"buffer=%p length=%zu max_short=%zd rndv_thresh=%zu "
"zcopy_thresh=%zu",
req, req->send.datatype, req->send.buffer, req->send.length,
max_short, rndv_thresh, zcopy_thresh);
status = ucp_request_send_start(req, max_short, zcopy_thresh, seg_size,
rndv_thresh, proto);
if (ucs_unlikely(status != UCS_OK)) {
if (status == UCS_ERR_NO_PROGRESS) {
ucs_assert(req->send.length >= rndv_thresh);
/* RMA/AM rendezvous */
status = ucp_tag_send_start_rndv(req);
}
if (status != UCS_OK) {
return UCS_STATUS_PTR(status);
}
}
ucp_request_send_tag_stat(req);
/*
* Start the request.
* If it is completed immediately, release the request and return the status.
* Otherwise, return the request.
*/
status = ucp_request_send(req);
if (req->flags & UCP_REQUEST_FLAG_COMPLETED) {
ucs_trace_req("releasing send request %p, returning status %s", req,
ucs_status_string(status));
ucp_request_put(req);
return UCS_STATUS_PTR(status);
}
ucp_request_set_callback(req, send.cb, cb)
ucs_trace_req("returning send request %p", req);
return req + 1;
}
static ucs_status_t ucp_tag_req_start_contig(ucp_request_t *req, size_t count,
ssize_t max_short, size_t zcopy_thresh,
size_t rndv_thresh,
const ucp_proto_t *proto)
{
ucp_ep_config_t *config = ucp_ep_config(req->send.ep);
size_t only_hdr_size = proto->only_hdr_size;
ucs_status_t status;
size_t max_zcopy;
ssize_t length;
length = ucp_contig_dt_length(req->send.datatype, count);
req->send.length = length;
if (length <= max_short) {
/* short */
req->send.uct.func = proto->contig_short;
} else if (length >= rndv_thresh) {
/* rendezvous */
status = ucp_tag_send_start_rndv(req);
if (status != UCS_OK) {
return status;
}
} else if (length < zcopy_thresh) {
/* bcopy */
if (req->send.length <= config->max_am_bcopy - only_hdr_size) {
req->send.uct.func = proto->bcopy_single;
} else {
req->send.uct.func = proto->bcopy_multi;
}
} else {
/* eager zcopy */
status = ucp_request_send_buffer_reg(req, ucp_ep_get_am_lane(req->send.ep));
if (status != UCS_OK) {
return status;
}
req->send.uct_comp.func = proto->contig_zcopy_completion;
max_zcopy = config->max_am_zcopy;
if (req->send.length <= max_zcopy - only_hdr_size) {
req->send.uct_comp.count = 1;
req->send.uct.func = proto->contig_zcopy_single;
} else {
/* calculate number of zcopy fragments */
req->send.uct_comp.count = 1 +
(length + proto->first_hdr_size - proto->mid_hdr_size - 1) /
(max_zcopy - proto->mid_hdr_size);
req->send.uct.func = proto->contig_zcopy_multi;
}
}
return UCS_OK;
}
static ucs_status_t
ucp_tag_send_start_req(ucp_ep_h ep, const void *buffer, size_t count,
ucp_datatype_t datatype, ucp_tag_t tag,
ucp_request_t *req)
{
size_t rndv_thresh = ep->worker->context->config.rndv_thresh;
ucp_dt_generic_t *dt_gen;
void *state;
req->send.ep = ep;
req->send.buffer = buffer;
req->send.count = count;
req->send.datatype = datatype;
req->send.state.offset = 0;
req->send.tag = tag;
switch (datatype & UCP_DATATYPE_CLASS_MASK) {
case UCP_DATATYPE_CONTIG:
/* TODO check for zero-copy */
req->send.length = ucp_contig_dt_length(datatype, count);
if (req->send.length <= rndv_thresh) {
req->send.uct.func = ucp_tag_progress_eager_contig;
return UCS_OK;
}
break;
case UCP_DATATYPE_GENERIC:
dt_gen = ucp_dt_generic(datatype);
state = dt_gen->ops.start_pack(dt_gen->context, buffer, count);
req->send.state.dt.generic.state = state;
req->send.length = dt_gen->ops.packed_size(state);
if (req->send.length <= rndv_thresh) {
req->send.uct.func = ucp_tag_progress_eager_generic;
return UCS_OK;
}
break;
default:
return UCS_ERR_INVALID_PARAM;
}
return ucp_tag_send_start_rndv(req);
}
