static ucs_status_t ucp_tag_req_start_contig(ucp_request_t *req, size_t count,
ssize_t max_short, size_t zcopy_thresh,
size_t rndv_thresh,
const ucp_proto_t *proto)
{
ucp_ep_config_t *config = ucp_ep_config(req->send.ep);
size_t only_hdr_size = proto->only_hdr_size;
ucs_status_t status;
size_t max_zcopy;
ssize_t length;
length = ucp_contig_dt_length(req->send.datatype, count);
req->send.length = length;
if (length <= max_short) {
/* short */
req->send.uct.func = proto->contig_short;
} else if (length >= rndv_thresh) {
/* rendezvous */
status = ucp_tag_send_start_rndv(req);
if (status != UCS_OK) {
return status;
}
} else if (length < zcopy_thresh) {
/* bcopy */
if (req->send.length <= config->max_am_bcopy - only_hdr_size) {
req->send.uct.func = proto->bcopy_single;
} else {
req->send.uct.func = proto->bcopy_multi;
}
} else {
/* eager zcopy */
status = ucp_request_send_buffer_reg(req, ucp_ep_get_am_lane(req->send.ep));
if (status != UCS_OK) {
return status;
}
req->send.uct_comp.func = proto->contig_zcopy_completion;
max_zcopy = config->max_am_zcopy;
if (req->send.length <= max_zcopy - only_hdr_size) {
req->send.uct_comp.count = 1;
req->send.uct.func = proto->contig_zcopy_single;
} else {
/* calculate number of zcopy fragments */
req->send.uct_comp.count = 1 +
(length + proto->first_hdr_size - proto->mid_hdr_size - 1) /
(max_zcopy - proto->mid_hdr_size);
req->send.uct.func = proto->contig_zcopy_multi;
}
}
return UCS_OK;
}
static UCS_F_ALWAYS_INLINE ucs_status_t
ucp_tag_send_try(ucp_ep_h ep, const void *buffer, size_t count,
ucp_datatype_t datatype, ucp_tag_t tag)
{
size_t length;
if ((datatype & UCP_DATATYPE_CLASS_MASK) == UCP_DATATYPE_CONTIG) {
length = ucp_contig_dt_length(datatype, count);
if (length <= ep->config.max_short_egr) {
return ucp_tag_send_eager_short(ep, tag, buffer, length);
}
}
return UCS_ERR_NO_RESOURCE; /* Fallback to slower progress */
}
static ucs_status_t
ucp_tag_send_start_req(ucp_ep_h ep, const void *buffer, size_t count,
ucp_datatype_t datatype, ucp_tag_t tag,
ucp_request_t *req)
{
size_t rndv_thresh = ep->worker->context->config.rndv_thresh;
ucp_dt_generic_t *dt_gen;
void *state;
req->send.ep = ep;
req->send.buffer = buffer;
req->send.count = count;
req->send.datatype = datatype;
req->send.state.offset = 0;
req->send.tag = tag;
switch (datatype & UCP_DATATYPE_CLASS_MASK) {
case UCP_DATATYPE_CONTIG:
/* TODO check for zero-copy */
req->send.length = ucp_contig_dt_length(datatype, count);
if (req->send.length <= rndv_thresh) {
req->send.uct.func = ucp_tag_progress_eager_contig;
return UCS_OK;
}
break;
case UCP_DATATYPE_GENERIC:
dt_gen = ucp_dt_generic(datatype);
state = dt_gen->ops.start_pack(dt_gen->context, buffer, count);
req->send.state.dt.generic.state = state;
req->send.length = dt_gen->ops.packed_size(state);
if (req->send.length <= rndv_thresh) {
req->send.uct.func = ucp_tag_progress_eager_generic;
return UCS_OK;
}
break;
default:
return UCS_ERR_INVALID_PARAM;
}
return ucp_tag_send_start_rndv(req);
}
ucs_status_ptr_t ucp_tag_send_nb(ucp_ep_h ep, const void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag,
ucp_send_callback_t cb)
{
ucs_status_t status;
ucp_request_t *req;
size_t length;
ucs_trace_req("send_nb buffer %p count %zu tag %"PRIx64" to %s cb %p",
buffer, count, tag, ucp_ep_peer_name(ep), cb);
if (ucs_likely((datatype & UCP_DATATYPE_CLASS_MASK) == UCP_DATATYPE_CONTIG)) {
length = ucp_contig_dt_length(datatype, count);
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX,
"ucp_tag_send_nb (eager - start)",
buffer, length);
if (ucs_likely(length <= ucp_ep_config(ep)->max_eager_short)) {
status = ucp_tag_send_eager_short(ep, tag, buffer, length);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX,
"ucp_tag_send_nb (eager - finish)",
buffer, length);
return UCS_STATUS_PTR(status); /* UCS_OK also goes here */
}
}
}
req = ucp_request_get(ep->worker);
if (req == NULL) {
return UCS_STATUS_PTR(UCS_ERR_NO_MEMORY);
}
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX, "ucp_tag_send_nb", req,
ucp_dt_length(datatype, count, buffer, &req->send.state));
ucp_tag_send_req_init(req, ep, buffer, datatype, tag);
return ucp_tag_send_req(req, count,
ucp_ep_config(ep)->max_eager_short,
ucp_ep_config(ep)->zcopy_thresh,
ucp_ep_config(ep)->rndv_thresh,
cb, &ucp_tag_eager_proto);
}
