static UCS_F_ALWAYS_INLINE void ucp_tag_recv_request_completed(ucp_request_t *req, ucs_status_t status, ucp_tag_recv_info_t *info, const char *function) { ucs_trace_req("%s returning completed request %p (%p) stag 0x%"PRIx64" len %zu, %s", function, req, req + 1, info->sender_tag, info->length, ucs_status_string(status)); req->status = status; if ((req->flags |= UCP_REQUEST_FLAG_COMPLETED) & UCP_REQUEST_FLAG_RELEASED) { ucp_request_put(req); } UCS_PROFILE_REQUEST_EVENT(req, "complete_recv", 0); }
static UCS_F_ALWAYS_INLINE void ucp_tag_recv_request_completed(ucp_request_t *req, ucs_status_t status, ucp_tag_recv_info_t *info, const char *function) { ucs_trace_req("%s returning completed request %p (%p) stag 0x%"PRIx64" len %zu, %s", function, req, req + 1, info->sender_tag, info->length, ucs_status_string(status)); req->status = status; req->flags |= UCP_REQUEST_FLAG_COMPLETED; if (req->flags & UCP_REQUEST_FLAG_BLOCK_OFFLOAD) { --req->recv.worker->context->tm.sw_req_count; } UCS_PROFILE_REQUEST_EVENT(req, "complete_recv", 0); }
static UCS_F_ALWAYS_INLINE void ucp_eager_expected_handler(ucp_worker_t *worker, ucp_request_t *req, void *data, size_t recv_len, ucp_tag_t recv_tag, uint16_t flags) { ucs_trace_req("found req %p", req); UCS_PROFILE_REQUEST_EVENT(req, "eager_recv", recv_len); /* First fragment fills the receive information */ UCP_WORKER_STAT_EAGER_MSG(worker, flags); UCP_WORKER_STAT_EAGER_CHUNK(worker, EXP); req->recv.tag.info.sender_tag = recv_tag; /* Cancel req in transport if it was offloaded, * because it arrived either: * 1) via SW TM (e. g. peer doesn't support offload) * 2) as unexpected via HW TM */ ucp_tag_offload_try_cancel(worker, req, UCP_TAG_OFFLOAD_CANCEL_FORCE | UCP_TAG_OFFLOAD_CANCEL_DEREG); }
static UCS_F_ALWAYS_INLINE void ucp_tag_recv_common(ucp_worker_h worker, void *buffer, size_t count, uintptr_t datatype, ucp_tag_t tag, ucp_tag_t tag_mask, ucp_request_t *req, uint32_t req_flags, ucp_tag_recv_callback_t cb, ucp_recv_desc_t *rdesc, const char *debug_name) { unsigned common_flags = UCP_REQUEST_FLAG_RECV | UCP_REQUEST_FLAG_EXPECTED; ucp_eager_first_hdr_t *eagerf_hdr; ucp_request_queue_t *req_queue; uct_memory_type_t mem_type; size_t hdr_len, recv_len; ucs_status_t status; uint64_t msg_id; ucp_trace_req(req, "%s buffer %p dt 0x%lx count %zu tag %"PRIx64"/%"PRIx64, debug_name, buffer, datatype, count, tag, tag_mask); /* First, check the fast path case - single fragment * in this case avoid initializing most of request fields * */ if (ucs_likely((rdesc != NULL) && (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_ONLY))) { UCS_PROFILE_REQUEST_EVENT(req, "eager_only_match", 0); UCP_WORKER_STAT_EAGER_MSG(worker, rdesc->flags); UCP_WORKER_STAT_EAGER_CHUNK(worker, UNEXP); if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_SYNC)) { ucp_tag_eager_sync_send_ack(worker, rdesc + 1, rdesc->flags); } req->flags = UCP_REQUEST_FLAG_RECV | req_flags; hdr_len = rdesc->payload_offset; recv_len = rdesc->length - hdr_len; req->recv.tag.info.sender_tag = ucp_rdesc_get_tag(rdesc); req->recv.tag.info.length = recv_len; ucp_memory_type_detect_mds(worker->context, buffer, recv_len, &mem_type); status = ucp_dt_unpack_only(worker, buffer, count, datatype, mem_type, (void*)(rdesc + 1) + hdr_len, recv_len, 1); ucp_recv_desc_release(rdesc); if (req_flags & UCP_REQUEST_FLAG_CALLBACK) { cb(req + 1, status, &req->recv.tag.info); } ucp_tag_recv_request_completed(req, status, &req->recv.tag.info, debug_name); return; } /* Initialize receive request */ req->status = UCS_OK; req->recv.worker = worker; req->recv.buffer = buffer; req->recv.datatype = datatype; ucp_dt_recv_state_init(&req->recv.state, buffer, datatype, count); if (!UCP_DT_IS_CONTIG(datatype)) { common_flags |= UCP_REQUEST_FLAG_BLOCK_OFFLOAD; } req->flags = common_flags | req_flags; req->recv.length = ucp_dt_length(datatype, count, buffer, &req->recv.state); ucp_memory_type_detect_mds(worker->context, buffer, req->recv.length, &mem_type); req->recv.mem_type = mem_type; req->recv.tag.tag = tag; req->recv.tag.tag_mask = tag_mask; req->recv.tag.cb = cb; if (ucs_log_is_enabled(UCS_LOG_LEVEL_TRACE_REQ)) { req->recv.tag.info.sender_tag = 0; } if (ucs_unlikely(rdesc == NULL)) { /* If not found on unexpected, wait until it arrives. * If was found but need this receive request for later completion, save it */ req_queue = ucp_tag_exp_get_queue(&worker->tm, tag, tag_mask); /* If offload supported, post this tag to transport as well. * TODO: need to distinguish the cases when posting is not needed. */ ucp_tag_offload_try_post(worker, req, req_queue); ucp_tag_exp_push(&worker->tm, req_queue, req); ucs_trace_req("%s returning expected request %p (%p)", debug_name, req, req + 1); return; } /* Check rendezvous case */ if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV)) { ucp_rndv_matched(worker, req, (void*)(rdesc + 1)); UCP_WORKER_STAT_RNDV(worker, UNEXP); ucp_recv_desc_release(rdesc); return; } if (ucs_unlikely(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER_SYNC)) { ucp_tag_eager_sync_send_ack(worker, rdesc + 1, rdesc->flags); } UCP_WORKER_STAT_EAGER_MSG(worker, rdesc->flags); ucs_assert(rdesc->flags & UCP_RECV_DESC_FLAG_EAGER); eagerf_hdr = (void*)(rdesc + 1); req->recv.tag.info.sender_tag = ucp_rdesc_get_tag(rdesc); req->recv.tag.info.length = req->recv.tag.remaining = eagerf_hdr->total_len; /* process first fragment */ UCP_WORKER_STAT_EAGER_CHUNK(worker, UNEXP); msg_id = eagerf_hdr->msg_id; status = ucp_tag_recv_request_process_rdesc(req, rdesc, 0); ucs_assert(status == UCS_INPROGRESS); /* process additional fragments */ ucp_tag_frag_list_process_queue(&worker->tm, req, msg_id UCS_STATS_ARG(UCP_WORKER_STAT_TAG_RX_EAGER_CHUNK_UNEXP)); }
static UCS_F_ALWAYS_INLINE ucs_status_t ucp_tag_search_unexp(ucp_worker_h worker, void *buffer, size_t buffer_size, ucp_datatype_t datatype, ucp_tag_t tag, uint64_t tag_mask, ucp_request_t *req, ucp_tag_recv_info_t *info, ucp_tag_recv_callback_t cb, ucp_recv_desc_t *first_rdesc, unsigned *save_rreq) { ucp_context_h context = worker->context; ucp_recv_desc_t *rdesc, *next; ucs_list_link_t *list; ucs_status_t status; ucp_tag_t recv_tag; unsigned flags; int i_list; /* fast check of global unexpected queue */ if (ucs_list_is_empty(&context->tm.unexpected.all)) { return UCS_INPROGRESS; } if (first_rdesc == NULL) { if (tag_mask == UCP_TAG_MASK_FULL) { list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag); if (ucs_list_is_empty(list)) { return UCS_INPROGRESS; } i_list = UCP_RDESC_HASH_LIST; } else { list = &context->tm.unexpected.all; i_list = UCP_RDESC_ALL_LIST; } rdesc = ucs_list_head(list, ucp_recv_desc_t, list[i_list]); } else { ucs_assert(tag_mask == UCP_TAG_MASK_FULL); list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag); i_list = UCP_RDESC_HASH_LIST; rdesc = first_rdesc; } do { recv_tag = ucp_rdesc_get_tag(rdesc); flags = rdesc->flags; ucs_trace_req("searching for %"PRIx64"/%"PRIx64"/%"PRIx64" offset %zu, " "checking desc %p %"PRIx64" %c%c%c%c%c", tag, tag_mask, info->sender_tag, req->recv.state.offset, rdesc, recv_tag, (flags & UCP_RECV_DESC_FLAG_FIRST) ? 'f' : '-', (flags & UCP_RECV_DESC_FLAG_LAST) ? 'l' : '-', (flags & UCP_RECV_DESC_FLAG_EAGER) ? 'e' : '-', (flags & UCP_RECV_DESC_FLAG_SYNC) ? 's' : '-', (flags & UCP_RECV_DESC_FLAG_RNDV) ? 'r' : '-'); if (ucp_tag_recv_is_match(recv_tag, flags, tag, tag_mask, req->recv.state.offset, info->sender_tag)) { ucp_tag_log_match(recv_tag, rdesc->length - rdesc->hdr_len, req, tag, tag_mask, req->recv.state.offset, "unexpected"); ucp_tag_unexp_remove(rdesc); if (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER) { UCS_PROFILE_REQUEST_EVENT(req, "eager_match", 0); status = ucp_eager_unexp_match(worker, rdesc, recv_tag, flags, buffer, buffer_size, datatype, &req->recv.state, info); ucs_trace_req("release receive descriptor %p", rdesc); if (status != UCS_INPROGRESS) { goto out_release_desc; } next = ucp_tag_unexp_list_next(rdesc, i_list); ucp_tag_unexp_desc_release(rdesc); rdesc = next; } else { ucs_assert_always(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV); *save_rreq = 0; req->recv.buffer = buffer; req->recv.length = buffer_size; req->recv.datatype = datatype; req->recv.cb = cb; ucp_rndv_matched(worker, req, (void*)(rdesc + 1)); UCP_WORKER_STAT_RNDV(worker, UNEXP); status = UCS_INPROGRESS; goto out_release_desc; } } else { rdesc = ucp_tag_unexp_list_next(rdesc, i_list); } } while (&rdesc->list[i_list] != list); return UCS_INPROGRESS; out_release_desc: ucp_tag_unexp_desc_release(rdesc); return status; }