void __ucs_twheel_sweep(ucs_twheel_t *t, ucs_time_t current_time) { ucs_wtimer_t *timer; uint64_t slot; slot = (current_time - t->now) >> t->res_order; t->now = current_time; if (ucs_unlikely(slot >= t->num_slots)) { slot = t->num_slots - 1; } slot = (t->current + slot) % t->num_slots; for (; t->current != slot; t->current = (t->current+1) % t->num_slots) { while (!ucs_list_is_empty(&t->wheel[t->current])) { timer = ucs_list_extract_head(&t->wheel[t->current], ucs_wtimer_t, list); timer->is_active = 0; ucs_invoke_callback(&timer->cb); } } }
static UCS_F_ALWAYS_INLINE ucs_status_t ucp_tag_search_unexp(ucp_worker_h worker, void *buffer, size_t buffer_size, ucp_datatype_t datatype, ucp_tag_t tag, uint64_t tag_mask, ucp_request_t *req, ucp_tag_recv_info_t *info, ucp_tag_recv_callback_t cb, ucp_recv_desc_t *first_rdesc, unsigned *save_rreq) { ucp_context_h context = worker->context; ucp_recv_desc_t *rdesc, *next; ucs_list_link_t *list; ucs_status_t status; ucp_tag_t recv_tag; unsigned flags; int i_list; /* fast check of global unexpected queue */ if (ucs_list_is_empty(&context->tm.unexpected.all)) { return UCS_INPROGRESS; } if (first_rdesc == NULL) { if (tag_mask == UCP_TAG_MASK_FULL) { list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag); if (ucs_list_is_empty(list)) { return UCS_INPROGRESS; } i_list = UCP_RDESC_HASH_LIST; } else { list = &context->tm.unexpected.all; i_list = UCP_RDESC_ALL_LIST; } rdesc = ucs_list_head(list, ucp_recv_desc_t, list[i_list]); } else { ucs_assert(tag_mask == UCP_TAG_MASK_FULL); list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag); i_list = UCP_RDESC_HASH_LIST; rdesc = first_rdesc; } do { recv_tag = ucp_rdesc_get_tag(rdesc); flags = rdesc->flags; ucs_trace_req("searching for %"PRIx64"/%"PRIx64"/%"PRIx64" offset %zu, " "checking desc %p %"PRIx64" %c%c%c%c%c", tag, tag_mask, info->sender_tag, req->recv.state.offset, rdesc, recv_tag, (flags & UCP_RECV_DESC_FLAG_FIRST) ? 'f' : '-', (flags & UCP_RECV_DESC_FLAG_LAST) ? 'l' : '-', (flags & UCP_RECV_DESC_FLAG_EAGER) ? 'e' : '-', (flags & UCP_RECV_DESC_FLAG_SYNC) ? 's' : '-', (flags & UCP_RECV_DESC_FLAG_RNDV) ? 'r' : '-'); if (ucp_tag_recv_is_match(recv_tag, flags, tag, tag_mask, req->recv.state.offset, info->sender_tag)) { ucp_tag_log_match(recv_tag, rdesc->length - rdesc->hdr_len, req, tag, tag_mask, req->recv.state.offset, "unexpected"); ucp_tag_unexp_remove(rdesc); if (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER) { UCS_PROFILE_REQUEST_EVENT(req, "eager_match", 0); status = ucp_eager_unexp_match(worker, rdesc, recv_tag, flags, buffer, buffer_size, datatype, &req->recv.state, info); ucs_trace_req("release receive descriptor %p", rdesc); if (status != UCS_INPROGRESS) { goto out_release_desc; } next = ucp_tag_unexp_list_next(rdesc, i_list); ucp_tag_unexp_desc_release(rdesc); rdesc = next; } else { ucs_assert_always(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV); *save_rreq = 0; req->recv.buffer = buffer; req->recv.length = buffer_size; req->recv.datatype = datatype; req->recv.cb = cb; ucp_rndv_matched(worker, req, (void*)(rdesc + 1)); UCP_WORKER_STAT_RNDV(worker, UNEXP); status = UCS_INPROGRESS; goto out_release_desc; } } else { rdesc = ucp_tag_unexp_list_next(rdesc, i_list); } } while (&rdesc->list[i_list] != list); return UCS_INPROGRESS; out_release_desc: ucp_tag_unexp_desc_release(rdesc); return status; }