static UCS_F_ALWAYS_INLINE ucs_status_ptr_t
ucp_tag_send_req(ucp_request_t *req, size_t count,
const ucp_ep_msg_config_t* msg_config,
size_t rndv_rma_thresh, size_t rndv_am_thresh,
ucp_send_callback_t cb, const ucp_proto_t *proto)
{
size_t seg_size = (msg_config->max_bcopy - proto->only_hdr_size);
size_t rndv_thresh = ucp_tag_get_rndv_threshold(req, count,
msg_config->max_iov,
rndv_rma_thresh,
rndv_am_thresh, seg_size);
size_t zcopy_thresh = ucp_proto_get_zcopy_threshold(req, msg_config, count,
rndv_thresh);
ssize_t max_short = ucp_proto_get_short_max(req, msg_config);
ucs_status_t status;
ucs_trace_req("select tag request(%p) progress algorithm datatype=%lx "
"buffer=%p length=%zu max_short=%zd rndv_thresh=%zu "
"zcopy_thresh=%zu",
req, req->send.datatype, req->send.buffer, req->send.length,
max_short, rndv_thresh, zcopy_thresh);
status = ucp_request_send_start(req, max_short, zcopy_thresh, seg_size,
rndv_thresh, proto);
if (ucs_unlikely(status != UCS_OK)) {
if (status == UCS_ERR_NO_PROGRESS) {
ucs_assert(req->send.length >= rndv_thresh);
/* RMA/AM rendezvous */
status = ucp_tag_send_start_rndv(req);
}
if (status != UCS_OK) {
return UCS_STATUS_PTR(status);
}
}
ucp_request_send_tag_stat(req);
/*
* Start the request.
* If it is completed immediately, release the request and return the status.
* Otherwise, return the request.
*/
status = ucp_request_send(req);
if (req->flags & UCP_REQUEST_FLAG_COMPLETED) {
ucs_trace_req("releasing send request %p, returning status %s", req,
ucs_status_string(status));
ucp_request_put(req);
return UCS_STATUS_PTR(status);
}
ucp_request_set_callback(req, send.cb, cb)
ucs_trace_req("returning send request %p", req);
return req + 1;
}
static ucs_status_ptr_t ucp_disconnect_nb_internal(ucp_ep_h ep)
{
ucs_status_t status;
ucp_request_t *req;
ucs_debug("disconnect ep %p", ep);
req = ucs_mpool_get(&ep->worker->req_mp);
if (req == NULL) {
return UCS_STATUS_PTR(UCS_ERR_NO_MEMORY);
}
/*
* Flush operation can be queued on the pending queue of only one of the
* lanes (indicated by req->send.lane) and scheduled for completion on any
* number of lanes. req->send.uct_comp.count keeps track of how many lanes
* are not flushed yet, and when it reaches zero, it means all lanes are
* flushed. req->send.flush.lanes keeps track of which lanes we still have
* to start flush on.
* If a flush is completed from a pending/completion callback, we need to
* schedule slow-path callback to release the endpoint later, since a UCT
* endpoint cannot be released from pending/completion callback context.
*/
req->flags = 0;
req->status = UCS_OK;
req->send.ep = ep;
req->send.flush.flushed_cb = ucp_ep_disconnected;
req->send.flush.lanes = UCS_MASK(ucp_ep_num_lanes(ep));
req->send.flush.cbq_elem.cb = ucp_ep_flushed_slow_path_callback;
req->send.flush.cbq_elem_on = 0;
req->send.lane = UCP_NULL_LANE;
req->send.uct.func = ucp_ep_flush_progress_pending;
req->send.uct_comp.func = ucp_ep_flush_completion;
req->send.uct_comp.count = ucp_ep_num_lanes(ep);
ucp_ep_flush_progress(req);
if (req->send.uct_comp.count == 0) {
status = req->status;
ucp_ep_disconnected(req);
ucs_trace_req("ep %p: releasing flush request %p, returning status %s",
ep, req, ucs_status_string(status));
ucs_mpool_put(req);
return UCS_STATUS_PTR(status);
}
ucs_trace_req("ep %p: return inprogress flush request %p (%p)", ep, req,
req + 1);
return req + 1;
}
static inline ucs_status_ptr_t
ucp_tag_send_req(ucp_request_t *req, size_t count, ssize_t max_short,
size_t zcopy_thresh, size_t rndv_thresh, ucp_send_callback_t cb,
const ucp_proto_t *proto)
{
ucs_status_t status;
switch (req->send.datatype & UCP_DATATYPE_CLASS_MASK) {
case UCP_DATATYPE_CONTIG:
status = ucp_tag_req_start_contig(req, count, max_short, zcopy_thresh,
rndv_thresh, proto);
if (status != UCS_OK) {
return UCS_STATUS_PTR(status);
}
break;
case UCP_DATATYPE_IOV:
status = ucp_tag_req_start_iov(req, count, max_short, zcopy_thresh,
rndv_thresh, proto);
if (status != UCS_OK) {
return UCS_STATUS_PTR(status);
}
break;
case UCP_DATATYPE_GENERIC:
ucp_tag_req_start_generic(req, count, rndv_thresh, proto);
break;
default:
ucs_error("Invalid data type");
return UCS_STATUS_PTR(UCS_ERR_INVALID_PARAM);
}
/*
* Start the request.
* If it is completed immediately, release the request and return the status.
* Otherwise, return the request.
*/
status = ucp_request_start_send(req);
if (req->flags & UCP_REQUEST_FLAG_COMPLETED) {
ucs_trace_req("releasing send request %p, returning status %s", req,
ucs_status_string(status));
ucs_mpool_put(req);
return UCS_STATUS_PTR(status);
}
ucs_trace_req("returning send request %p", req);
req->send.cb = cb;
return req + 1;
}
static UCS_F_ALWAYS_INLINE ucs_status_t
ucp_tag_recv_common(ucp_worker_h worker, void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag, ucp_tag_t tag_mask,
ucp_request_t *req, uint16_t req_flags, ucp_tag_recv_callback_t cb,
ucp_recv_desc_t *rdesc, const char *debug_name)
{
unsigned save_rreq = 1;
ucs_queue_head_t *queue;
ucp_context_h context;
ucs_status_t status;
size_t buffer_size;
ucp_tag_recv_request_init(req, worker, buffer, count, datatype, req_flags);
buffer_size = ucp_dt_length(datatype, count, buffer, &req->recv.state);
ucs_trace_req("%s buffer %p buffer_size %zu tag %"PRIx64"/%"PRIx64, debug_name,
buffer, buffer_size, tag, tag_mask);
/* First, search in unexpected list */
status = ucp_tag_search_unexp(worker, buffer, buffer_size, datatype, tag,
tag_mask, req, &req->recv.info, cb, rdesc,
&save_rreq);
if (status != UCS_INPROGRESS) {
if (req_flags & UCP_REQUEST_FLAG_CALLBACK) {
cb(req + 1, status, &req->recv.info);
}
ucp_tag_recv_request_completed(req, status, &req->recv.info, debug_name);
} else if (save_rreq) {
/* If not found on unexpected, wait until it arrives.
* If was found but need this receive request for later completion, save it */
context = worker->context;
queue = ucp_tag_exp_get_queue(&context->tm, tag, tag_mask);
req->recv.buffer = buffer;
req->recv.length = buffer_size;
req->recv.datatype = datatype;
req->recv.tag = tag;
req->recv.tag_mask = tag_mask;
req->recv.cb = cb;
ucp_tag_exp_push(&context->tm, queue, req);
/* If offload supported, post this tag to transport as well.
* TODO: need to distinguish the cases when posting is not needed. */
ucp_tag_offload_try_post(worker->context, req);
ucs_trace_req("%s returning expected request %p (%p)", debug_name, req,
req + 1);
}
return status;
}
static UCS_F_ALWAYS_INLINE void
ucp_request_release_common(void *request, uint8_t cb_flag, const char *debug_name)
{
ucp_request_t *req = (ucp_request_t*)request - 1;
ucp_worker_h UCS_V_UNUSED worker = ucs_container_of(ucs_mpool_obj_owner(req),
ucp_worker_t, req_mp);
uint16_t flags;
UCP_THREAD_CS_ENTER_CONDITIONAL(&worker->mt_lock);
flags = req->flags;
ucs_trace_req("%s request %p (%p) "UCP_REQUEST_FLAGS_FMT, debug_name,
req, req + 1, UCP_REQUEST_FLAGS_ARG(flags));
ucs_assert(!(flags & UCP_REQUEST_DEBUG_FLAG_EXTERNAL));
ucs_assert(!(flags & UCP_REQUEST_FLAG_RELEASED));
if (ucs_likely(flags & UCP_REQUEST_FLAG_COMPLETED)) {
ucp_request_put(req);
} else {
req->flags = (flags | UCP_REQUEST_FLAG_RELEASED) & ~cb_flag;
}
UCP_THREAD_CS_EXIT_CONDITIONAL(&worker->mt_lock);
}
void ucp_tag_eager_sync_send_ack(ucp_worker_h worker, void *hdr, uint16_t flags)
{
ucp_eager_sync_hdr_t *eagers_hdr;
ucp_request_hdr_t *reqhdr;
ucp_request_t *req;
ucs_assert(flags & UCP_RECV_DESC_FLAG_EAGER_SYNC);
if (flags & UCP_RECV_DESC_FLAG_EAGER_OFFLOAD) {
eagers_hdr = hdr;
ucp_tag_offload_eager_sync_send_ack(worker,
eagers_hdr->req.sender_uuid,
eagers_hdr->super.super.tag);
return;
}
if (flags & UCP_RECV_DESC_FLAG_EAGER_ONLY) {
reqhdr = &((ucp_eager_sync_hdr_t*)hdr)->req;
} else /* first */ {
reqhdr = &((ucp_eager_sync_first_hdr_t*)hdr)->req;
}
ucs_assert(reqhdr->reqptr != 0);
ucs_trace_req("send_sync_ack sender_uuid %"PRIx64" remote_request 0x%lx",
reqhdr->sender_uuid, reqhdr->reqptr);
req = ucp_worker_allocate_reply(worker, reqhdr->sender_uuid);
req->send.uct.func = ucp_proto_progress_am_bcopy_single;
req->send.proto.am_id = UCP_AM_ID_EAGER_SYNC_ACK;
req->send.proto.remote_request = reqhdr->reqptr;
req->send.proto.status = UCS_OK;
req->send.proto.comp_cb = ucp_request_put;
ucp_request_send(req);
}
ucs_status_ptr_t ucp_tag_send_sync_nb(ucp_ep_h ep, const void *buffer, size_t count,
ucp_datatype_t datatype, ucp_tag_t tag,
ucp_send_callback_t cb)
{
ucp_request_t *req;
ucs_trace_req("send_sync_nb buffer %p count %zu tag %"PRIx64" to %s cb %p",
buffer, count, tag, ucp_ep_peer_name(ep), cb);
req = ucp_request_get(ep->worker);
if (req == NULL) {
return UCS_STATUS_PTR(UCS_ERR_NO_MEMORY);
}
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX, "ucp_tag_send_sync_nb", req,
ucp_dt_length(datatype, count, buffer, &req->send.state));
/* Remote side needs to send reply, so have it connect to us */
ucp_ep_connect_remote(ep);
ucp_tag_send_req_init(req, ep, buffer, datatype, tag);
return ucp_tag_send_req(req, count,
-1, /* disable short method */
ucp_ep_config(ep)->sync_zcopy_thresh,
ucp_ep_config(ep)->sync_rndv_thresh,
cb, &ucp_tag_eager_sync_proto);
}
static UCS_F_NOINLINE
ucs_status_ptr_t ucp_tag_send_slow(ucp_ep_h ep, const void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag,
ucp_send_callback_t cb)
{
ucp_request_t *req;
ucs_status_t status;
req = ucs_mpool_get_inline(&ep->worker->req_mp);
if (req == NULL) {
return UCS_STATUS_PTR(UCS_ERR_NO_MEMORY);
}
VALGRIND_MAKE_MEM_DEFINED(req + 1, ep->worker->context->config.request.size);
req->flags = 0;
req->cb.send = cb;
status = ucp_tag_send_start_req(ep, buffer, count, datatype, tag, req);
if (status != UCS_OK) {
return UCS_STATUS_PTR(status); /* UCS_OK also goes here */
}
if (!(req->flags & UCP_REQUEST_FLAG_COMPLETED)) {
ucp_ep_add_pending(ep, ep->uct_ep, req);
ucp_worker_progress(ep->worker);
}
ucs_trace_req("send_nb returning request %p", req);
return req + 1;
}
static UCS_F_ALWAYS_INLINE void
ucp_tag_recv_request_completed(ucp_request_t *req, ucs_status_t status,
ucp_tag_recv_info_t *info, const char *function)
{
ucs_trace_req("%s returning completed request %p (%p) stag 0x%"PRIx64" len %zu, %s",
function, req, req + 1, info->sender_tag, info->length,
ucs_status_string(status));
req->status = status;
if ((req->flags |= UCP_REQUEST_FLAG_COMPLETED) & UCP_REQUEST_FLAG_RELEASED) {
ucp_request_put(req);
}
UCS_PROFILE_REQUEST_EVENT(req, "complete_recv", 0);
}
static UCS_F_ALWAYS_INLINE void
ucp_tag_recv_request_completed(ucp_request_t *req, ucs_status_t status,
ucp_tag_recv_info_t *info, const char *function)
{
ucs_trace_req("%s returning completed request %p (%p) stag 0x%"PRIx64" len %zu, %s",
function, req, req + 1, info->sender_tag, info->length,
ucs_status_string(status));
req->status = status;
req->flags |= UCP_REQUEST_FLAG_COMPLETED;
if (req->flags & UCP_REQUEST_FLAG_BLOCK_OFFLOAD) {
--req->recv.worker->context->tm.sw_req_count;
}
UCS_PROFILE_REQUEST_EVENT(req, "complete_recv", 0);
}
void ucp_tag_eager_sync_send_ack(ucp_worker_h worker, uint64_t sender_uuid,
uintptr_t remote_request)
{
ucp_request_t *req;
ucs_trace_req("send_sync_ack sender_uuid %"PRIx64" remote_request 0x%lx",
sender_uuid, remote_request);
req = ucp_worker_allocate_reply(worker, sender_uuid);
req->send.uct.func = ucp_proto_progress_am_bcopy_single;
req->send.proto.am_id = UCP_AM_ID_EAGER_SYNC_ACK;
req->send.proto.remote_request = remote_request;
req->send.proto.status = UCS_OK;
ucp_request_start_send(req);
}
ucs_status_ptr_t ucp_tag_send_nb(ucp_ep_h ep, const void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag,
ucp_send_callback_t cb)
{
ucs_status_t status;
ucs_trace_req("send_nb buffer %p count %zu tag %"PRIx64" to %s cb %p",
buffer, count, tag, ucp_ep_peer_name(ep), cb);
status = ucp_tag_send_try(ep, buffer, count, datatype, tag);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
return UCS_STATUS_PTR(status); /* UCS_OK also goes here */
}
return ucp_tag_send_slow(ep, buffer, count, datatype, tag, cb);
}
static void ucp_ep_flushed_slow_path_callback(ucs_callbackq_slow_elem_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.flush.cbq_elem);
ucp_ep_h ep = req->send.ep;
ucs_assert(!(req->flags & UCP_REQUEST_FLAG_COMPLETED));
ucs_trace("flush req %p ep %p remove from uct_worker %p", req, ep,
ep->worker->uct);
ucp_ep_flush_slow_path_remove(req);
req->send.flush.flushed_cb(req);
/* Complete send request from here, to avoid releasing the request while
* slow-path element is still pending */
ucs_trace_req("completing flush request %p (%p) with status %s", req, req + 1,
ucs_status_string(req->status));
ucp_request_put(req, req->status);
}
ucs_status_ptr_t ucp_tag_send_nb(ucp_ep_h ep, const void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag,
ucp_send_callback_t cb)
{
ucs_status_t status;
ucp_request_t *req;
size_t length;
ucs_trace_req("send_nb buffer %p count %zu tag %"PRIx64" to %s cb %p",
buffer, count, tag, ucp_ep_peer_name(ep), cb);
if (ucs_likely((datatype & UCP_DATATYPE_CLASS_MASK) == UCP_DATATYPE_CONTIG)) {
length = ucp_contig_dt_length(datatype, count);
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX,
"ucp_tag_send_nb (eager - start)",
buffer, length);
if (ucs_likely(length <= ucp_ep_config(ep)->max_eager_short)) {
status = ucp_tag_send_eager_short(ep, tag, buffer, length);
if (ucs_likely(status != UCS_ERR_NO_RESOURCE)) {
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX,
"ucp_tag_send_nb (eager - finish)",
buffer, length);
return UCS_STATUS_PTR(status); /* UCS_OK also goes here */
}
}
}
req = ucp_request_get(ep->worker);
if (req == NULL) {
return UCS_STATUS_PTR(UCS_ERR_NO_MEMORY);
}
UCS_INSTRUMENT_RECORD(UCS_INSTRUMENT_TYPE_UCP_TX, "ucp_tag_send_nb", req,
ucp_dt_length(datatype, count, buffer, &req->send.state));
ucp_tag_send_req_init(req, ep, buffer, datatype, tag);
return ucp_tag_send_req(req, count,
ucp_ep_config(ep)->max_eager_short,
ucp_ep_config(ep)->zcopy_thresh,
ucp_ep_config(ep)->rndv_thresh,
cb, &ucp_tag_eager_proto);
}
static UCS_F_ALWAYS_INLINE void
ucp_eager_expected_handler(ucp_worker_t *worker, ucp_request_t *req,
void *data, size_t recv_len, ucp_tag_t recv_tag,
uint16_t flags)
{
ucs_trace_req("found req %p", req);
UCS_PROFILE_REQUEST_EVENT(req, "eager_recv", recv_len);
/* First fragment fills the receive information */
UCP_WORKER_STAT_EAGER_MSG(worker, flags);
UCP_WORKER_STAT_EAGER_CHUNK(worker, EXP);
req->recv.tag.info.sender_tag = recv_tag;
/* Cancel req in transport if it was offloaded,
* because it arrived either:
* 1) via SW TM (e. g. peer doesn't support offload)
* 2) as unexpected via HW TM */
ucp_tag_offload_try_cancel(worker, req,
UCP_TAG_OFFLOAD_CANCEL_FORCE |
UCP_TAG_OFFLOAD_CANCEL_DEREG);
}
static UCS_F_ALWAYS_INLINE void
ucp_tag_recv_common(ucp_worker_h worker, void *buffer, size_t count,
uintptr_t datatype, ucp_tag_t tag, ucp_tag_t tag_mask,
ucp_request_t *req, uint32_t req_flags, ucp_tag_recv_callback_t cb,
ucp_recv_desc_t *rdesc, const char *debug_name)
{
unsigned common_flags = UCP_REQUEST_FLAG_RECV | UCP_REQUEST_FLAG_EXPECTED;
ucp_eager_first_hdr_t *eagerf_hdr;
ucp_request_queue_t *req_queue;
uct_memory_type_t mem_type;
size_t hdr_len, recv_len;
ucs_status_t status;
uint64_t msg_id;
ucp_trace_req(req, "%s buffer %p dt 0x%lx count %zu tag %"PRIx64"/%"PRIx64,
debug_name, buffer, datatype, count, tag, tag_mask);
/* First, check the fast path case - single fragment
* in this case avoid initializing most of request fields
* */
if (ucs_likely((rdesc != NULL) && (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_ONLY))) {
UCS_PROFILE_REQUEST_EVENT(req, "eager_only_match", 0);
UCP_WORKER_STAT_EAGER_MSG(worker, rdesc->flags);
UCP_WORKER_STAT_EAGER_CHUNK(worker, UNEXP);
if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_SYNC)) {
ucp_tag_eager_sync_send_ack(worker, rdesc + 1, rdesc->flags);
}
req->flags = UCP_REQUEST_FLAG_RECV | req_flags;
hdr_len = rdesc->payload_offset;
recv_len = rdesc->length - hdr_len;
req->recv.tag.info.sender_tag = ucp_rdesc_get_tag(rdesc);
req->recv.tag.info.length = recv_len;
ucp_memory_type_detect_mds(worker->context, buffer, recv_len, &mem_type);
status = ucp_dt_unpack_only(worker, buffer, count, datatype, mem_type,
(void*)(rdesc + 1) + hdr_len, recv_len, 1);
ucp_recv_desc_release(rdesc);
if (req_flags & UCP_REQUEST_FLAG_CALLBACK) {
cb(req + 1, status, &req->recv.tag.info);
}
ucp_tag_recv_request_completed(req, status, &req->recv.tag.info,
debug_name);
return;
}
/* Initialize receive request */
req->status = UCS_OK;
req->recv.worker = worker;
req->recv.buffer = buffer;
req->recv.datatype = datatype;
ucp_dt_recv_state_init(&req->recv.state, buffer, datatype, count);
if (!UCP_DT_IS_CONTIG(datatype)) {
common_flags |= UCP_REQUEST_FLAG_BLOCK_OFFLOAD;
}
req->flags = common_flags | req_flags;
req->recv.length = ucp_dt_length(datatype, count, buffer,
&req->recv.state);
ucp_memory_type_detect_mds(worker->context, buffer, req->recv.length, &mem_type);
req->recv.mem_type = mem_type;
req->recv.tag.tag = tag;
req->recv.tag.tag_mask = tag_mask;
req->recv.tag.cb = cb;
if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE_REQ)) {
req->recv.tag.info.sender_tag = 0;
}
if (ucs_unlikely(rdesc == NULL)) {
/* If not found on unexpected, wait until it arrives.
* If was found but need this receive request for later completion, save it */
req_queue = ucp_tag_exp_get_queue(&worker->tm, tag, tag_mask);
/* If offload supported, post this tag to transport as well.
* TODO: need to distinguish the cases when posting is not needed. */
ucp_tag_offload_try_post(worker, req, req_queue);
ucp_tag_exp_push(&worker->tm, req_queue, req);
ucs_trace_req("%s returning expected request %p (%p)", debug_name, req,
req + 1);
return;
}
/* Check rendezvous case */
if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV)) {
ucp_rndv_matched(worker, req, (void*)(rdesc + 1));
UCP_WORKER_STAT_RNDV(worker, UNEXP);
ucp_recv_desc_release(rdesc);
return;
}
if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_SYNC)) {
ucp_tag_eager_sync_send_ack(worker, rdesc + 1, rdesc->flags);
}
UCP_WORKER_STAT_EAGER_MSG(worker, rdesc->flags);
ucs_assert(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER);
eagerf_hdr = (void*)(rdesc + 1);
req->recv.tag.info.sender_tag = ucp_rdesc_get_tag(rdesc);
req->recv.tag.info.length =
req->recv.tag.remaining = eagerf_hdr->total_len;
/* process first fragment */
UCP_WORKER_STAT_EAGER_CHUNK(worker, UNEXP);
msg_id = eagerf_hdr->msg_id;
status = ucp_tag_recv_request_process_rdesc(req, rdesc, 0);
ucs_assert(status == UCS_INPROGRESS);
/* process additional fragments */
ucp_tag_frag_list_process_queue(&worker->tm, req, msg_id
UCS_STATS_ARG(UCP_WORKER_STAT_TAG_RX_EAGER_CHUNK_UNEXP));
}
static UCS_F_ALWAYS_INLINE ucs_status_t
ucp_tag_search_unexp(ucp_worker_h worker, void *buffer, size_t buffer_size,
ucp_datatype_t datatype, ucp_tag_t tag, uint64_t tag_mask,
ucp_request_t *req, ucp_tag_recv_info_t *info,
ucp_tag_recv_callback_t cb, ucp_recv_desc_t *first_rdesc,
unsigned *save_rreq)
{
ucp_context_h context = worker->context;
ucp_recv_desc_t *rdesc, *next;
ucs_list_link_t *list;
ucs_status_t status;
ucp_tag_t recv_tag;
unsigned flags;
int i_list;
/* fast check of global unexpected queue */
if (ucs_list_is_empty(&context->tm.unexpected.all)) {
return UCS_INPROGRESS;
}
if (first_rdesc == NULL) {
if (tag_mask == UCP_TAG_MASK_FULL) {
list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag);
if (ucs_list_is_empty(list)) {
return UCS_INPROGRESS;
}
i_list = UCP_RDESC_HASH_LIST;
} else {
list = &context->tm.unexpected.all;
i_list = UCP_RDESC_ALL_LIST;
}
rdesc = ucs_list_head(list, ucp_recv_desc_t, list[i_list]);
} else {
ucs_assert(tag_mask == UCP_TAG_MASK_FULL);
list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag);
i_list = UCP_RDESC_HASH_LIST;
rdesc = first_rdesc;
}
do {
recv_tag = ucp_rdesc_get_tag(rdesc);
flags = rdesc->flags;
ucs_trace_req("searching for %"PRIx64"/%"PRIx64"/%"PRIx64" offset %zu, "
"checking desc %p %"PRIx64" %c%c%c%c%c",
tag, tag_mask, info->sender_tag, req->recv.state.offset,
rdesc, recv_tag,
(flags & UCP_RECV_DESC_FLAG_FIRST) ? 'f' : '-',
(flags & UCP_RECV_DESC_FLAG_LAST) ? 'l' : '-',
(flags & UCP_RECV_DESC_FLAG_EAGER) ? 'e' : '-',
(flags & UCP_RECV_DESC_FLAG_SYNC) ? 's' : '-',
(flags & UCP_RECV_DESC_FLAG_RNDV) ? 'r' : '-');
if (ucp_tag_recv_is_match(recv_tag, flags, tag, tag_mask,
req->recv.state.offset, info->sender_tag))
{
ucp_tag_log_match(recv_tag, rdesc->length - rdesc->hdr_len, req, tag,
tag_mask, req->recv.state.offset, "unexpected");
ucp_tag_unexp_remove(rdesc);
if (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER) {
UCS_PROFILE_REQUEST_EVENT(req, "eager_match", 0);
status = ucp_eager_unexp_match(worker, rdesc, recv_tag, flags,
buffer, buffer_size, datatype,
&req->recv.state, info);
ucs_trace_req("release receive descriptor %p", rdesc);
if (status != UCS_INPROGRESS) {
goto out_release_desc;
}
next = ucp_tag_unexp_list_next(rdesc, i_list);
ucp_tag_unexp_desc_release(rdesc);
rdesc = next;
} else {
ucs_assert_always(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV);
*save_rreq = 0;
req->recv.buffer = buffer;
req->recv.length = buffer_size;
req->recv.datatype = datatype;
req->recv.cb = cb;
ucp_rndv_matched(worker, req, (void*)(rdesc + 1));
UCP_WORKER_STAT_RNDV(worker, UNEXP);
status = UCS_INPROGRESS;
goto out_release_desc;
}
} else {
rdesc = ucp_tag_unexp_list_next(rdesc, i_list);
}
} while (&rdesc->list[i_list] != list);
return UCS_INPROGRESS;
out_release_desc:
ucp_tag_unexp_desc_release(rdesc);
return status;
}
