static ucs_status_t ucp_tag_eager_am_handler(void *arg, void *data, size_t length,
void *desc)
{
ucp_context_h context = arg;
ucp_recv_desc_t *rdesc = desc;
ucp_recv_request_t *rreq;
ucs_queue_iter_t iter;
ucp_tag_t tag;
ucs_assert(length >= sizeof(ucp_tag_t));
tag = *(ucp_tag_t*)data;
/* Search in expected queue */
iter = ucs_queue_iter_begin(&context->tag.expected);
while (!ucs_queue_iter_end(&context->tag.expected, iter)) {
rreq = ucs_container_of(*iter, ucp_recv_request_t, queue);
if (ucp_tag_is_match(tag, rreq->tag, rreq->tag_mask)) {
ucs_queue_del_iter(&context->tag.expected, iter);
rreq->status = ucp_tag_matched(rreq->buffer, rreq->length, tag,
data + sizeof(ucp_tag_t),
length - sizeof(ucp_tag_t),
&rreq->comp);
return UCS_OK;
}
}
if (data != rdesc + 1) {
memcpy(rdesc + 1, data, length);
}
rdesc->length = length;
ucs_queue_push(&context->tag.unexpected, &rdesc->queue);
return UCS_INPROGRESS;
}
ucs_status_t ucp_tag_recv(ucp_worker_h worker, void *buffer,
size_t length, ucp_tag_t tag, uint64_t tag_mask,
ucp_tag_recv_completion_t *comp)
{
ucp_context_h context = worker->context;
ucp_recv_request_t rreq;
ucs_queue_iter_t iter;
ucp_recv_desc_t *rdesc;
ucp_tag_t unexp_tag;
ucs_status_t status;
/* First, search in unexpected list */
iter = ucs_queue_iter_begin(&context->tag.unexpected);
while (!ucs_queue_iter_end(&context->tag.unexpected, iter)) {
rdesc = ucs_container_of(*iter, ucp_recv_desc_t, queue);
unexp_tag = *(ucp_tag_t*)(rdesc + 1);
if (ucp_tag_is_match(unexp_tag, tag, tag_mask)) {
ucs_queue_del_iter(&context->tag.unexpected, iter);
status = ucp_tag_matched(buffer, length, unexp_tag,
(void*)(rdesc + 1) + sizeof(ucp_tag_t),
rdesc->length - sizeof(ucp_tag_t),
comp);
uct_iface_release_am_desc(rdesc);
goto out;
}
iter = ucs_queue_iter_next(iter);
}
/* If not found on unexpected, wait until it arrives */
rreq.status = UCS_INPROGRESS;
rreq.buffer = buffer;
rreq.length = length;
rreq.tag = tag;
rreq.tag_mask = tag_mask;
ucs_queue_push(&context->tag.expected, &rreq.queue);
do {
uct_worker_progress(worker->uct);
/* coverity[loop_condition] */
} while (rreq.status == UCS_INPROGRESS);
*comp = rreq.comp;
status = rreq.status;
out:
return status;
}
static UCS_F_ALWAYS_INLINE unsigned
uct_cuda_copy_progress_event_queue(ucs_queue_head_t *event_queue, unsigned max_events)
{
unsigned count = 0;
cudaError_t result = cudaSuccess;
uct_cuda_copy_event_desc_t *cuda_event;
ucs_queue_iter_t iter;
ucs_queue_for_each_safe(cuda_event, iter, event_queue, queue) {
result = cudaEventQuery(cuda_event->event);
if (cudaSuccess != result) {
break;
}
ucs_queue_del_iter(event_queue, iter);
if (cuda_event->comp != NULL) {
uct_invoke_completion(cuda_event->comp, UCS_OK);
}
ucs_trace_poll("CUDA Event Done :%p", cuda_event);
ucs_mpool_put(cuda_event);
count++;
if (count >= max_events) {
break;
}
}
