void __ucs_twheel_sweep(ucs_twheel_t *t, ucs_time_t current_time)
{
ucs_wtimer_t *timer;
uint64_t slot;
slot = (current_time - t->now) >> t->res_order;
t->now = current_time;
if (ucs_unlikely(slot >= t->num_slots)) {
slot = t->num_slots - 1;
}
slot = (t->current + slot) % t->num_slots;
for (; t->current != slot; t->current = (t->current+1) % t->num_slots) {
while (!ucs_list_is_empty(&t->wheel[t->current])) {
timer = ucs_list_extract_head(&t->wheel[t->current], ucs_wtimer_t, list);
timer->is_active = 0;
ucs_invoke_callback(&timer->cb);
}
}
}
static UCS_F_ALWAYS_INLINE ucs_status_t
ucp_tag_search_unexp(ucp_worker_h worker, void *buffer, size_t buffer_size,
ucp_datatype_t datatype, ucp_tag_t tag, uint64_t tag_mask,
ucp_request_t *req, ucp_tag_recv_info_t *info,
ucp_tag_recv_callback_t cb, ucp_recv_desc_t *first_rdesc,
unsigned *save_rreq)
{
ucp_context_h context = worker->context;
ucp_recv_desc_t *rdesc, *next;
ucs_list_link_t *list;
ucs_status_t status;
ucp_tag_t recv_tag;
unsigned flags;
int i_list;
/* fast check of global unexpected queue */
if (ucs_list_is_empty(&context->tm.unexpected.all)) {
return UCS_INPROGRESS;
}
if (first_rdesc == NULL) {
if (tag_mask == UCP_TAG_MASK_FULL) {
list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag);
if (ucs_list_is_empty(list)) {
return UCS_INPROGRESS;
}
i_list = UCP_RDESC_HASH_LIST;
} else {
list = &context->tm.unexpected.all;
i_list = UCP_RDESC_ALL_LIST;
}
rdesc = ucs_list_head(list, ucp_recv_desc_t, list[i_list]);
} else {
ucs_assert(tag_mask == UCP_TAG_MASK_FULL);
list = ucp_tag_unexp_get_list_for_tag(&context->tm, tag);
i_list = UCP_RDESC_HASH_LIST;
rdesc = first_rdesc;
}
do {
recv_tag = ucp_rdesc_get_tag(rdesc);
flags = rdesc->flags;
ucs_trace_req("searching for %"PRIx64"/%"PRIx64"/%"PRIx64" offset %zu, "
"checking desc %p %"PRIx64" %c%c%c%c%c",
tag, tag_mask, info->sender_tag, req->recv.state.offset,
rdesc, recv_tag,
(flags & UCP_RECV_DESC_FLAG_FIRST) ? 'f' : '-',
(flags & UCP_RECV_DESC_FLAG_LAST) ? 'l' : '-',
(flags & UCP_RECV_DESC_FLAG_EAGER) ? 'e' : '-',
(flags & UCP_RECV_DESC_FLAG_SYNC) ? 's' : '-',
(flags & UCP_RECV_DESC_FLAG_RNDV) ? 'r' : '-');
if (ucp_tag_recv_is_match(recv_tag, flags, tag, tag_mask,
req->recv.state.offset, info->sender_tag))
{
ucp_tag_log_match(recv_tag, rdesc->length - rdesc->hdr_len, req, tag,
tag_mask, req->recv.state.offset, "unexpected");
ucp_tag_unexp_remove(rdesc);
if (rdesc->flags & UCP_RECV_DESC_FLAG_EAGER) {
UCS_PROFILE_REQUEST_EVENT(req, "eager_match", 0);
status = ucp_eager_unexp_match(worker, rdesc, recv_tag, flags,
buffer, buffer_size, datatype,
&req->recv.state, info);
ucs_trace_req("release receive descriptor %p", rdesc);
if (status != UCS_INPROGRESS) {
goto out_release_desc;
}
next = ucp_tag_unexp_list_next(rdesc, i_list);
ucp_tag_unexp_desc_release(rdesc);
rdesc = next;
} else {
ucs_assert_always(rdesc->flags & UCP_RECV_DESC_FLAG_RNDV);
*save_rreq = 0;
req->recv.buffer = buffer;
req->recv.length = buffer_size;
req->recv.datatype = datatype;
req->recv.cb = cb;
ucp_rndv_matched(worker, req, (void*)(rdesc + 1));
UCP_WORKER_STAT_RNDV(worker, UNEXP);
status = UCS_INPROGRESS;
goto out_release_desc;
}
} else {
rdesc = ucp_tag_unexp_list_next(rdesc, i_list);
}
} while (&rdesc->list[i_list] != list);
return UCS_INPROGRESS;
out_release_desc:
ucp_tag_unexp_desc_release(rdesc);
return status;
}
