static ucs_status_t uct_ugni_add_flush_comp(uct_ugni_ep_t *ep, unsigned flags,
uct_completion_t *comp)
{
uct_ugni_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_ugni_iface_t);
uct_ugni_flush_group_t *new_group, *present_group;
if (!uct_ugni_ep_can_send(ep)) {
return UCS_ERR_NO_RESOURCE;
}
if (NULL == comp) {
return UCS_INPROGRESS;
}
new_group = uct_ugni_new_flush_group(iface);
new_group->flush_comp.count = UCT_UGNI_INIT_FLUSH_REQ;
#ifdef DEBUG
new_group->flush_comp.func = NULL;
new_group->parent = NULL;
#endif
present_group = (uct_ugni_flush_group_t*)uct_ugni_safe_swap_pointers(&ep->flush_group,
(uintptr_t)new_group);
present_group->flush_comp.func = uct_ugni_flush_cb;
present_group->user_comp = comp;
present_group->parent = new_group;
uct_invoke_completion(&present_group->flush_comp, UCS_OK);
return UCS_INPROGRESS;
}
static void uct_ugni_flush_cb(uct_completion_t *self, ucs_status_t status)
{
uct_ugni_flush_group_t *group = ucs_container_of(self, uct_ugni_flush_group_t, flush_comp);
ucs_trace("group=%p, parent=%p, user_comp=%p", group, group->parent, group->user_comp);
uct_invoke_completion(group->user_comp, UCS_OK);
uct_ugni_check_flush(group->parent);
uct_ugni_put_flush_group(group);
}
void uct_ugni_progress(void *arg)
{
gni_cq_entry_t event_data = 0;
gni_post_descriptor_t *event_post_desc_ptr;
uct_ugni_base_desc_t *desc;
uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg;
gni_return_t ugni_rc;
ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data);
if (GNI_RC_NOT_DONE == ugni_rc) {
goto out;
}
if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) {
ucs_error("GNI_CqGetEvent falied. Error status %s %d ",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr);
if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) {
ucs_error("GNI_GetCompleted falied. Error status %s %d %d",
gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR);
return;
}
desc = (uct_ugni_base_desc_t *)event_post_desc_ptr;
ucs_trace_async("Completion received on %p", desc);
if (NULL != desc->comp_cb) {
uct_invoke_completion(desc->comp_cb, UCS_OK);
}
--iface->outstanding;
--desc->ep->outstanding;
if (ucs_likely(0 == desc->not_ready_to_free)) {
ucs_mpool_put(desc);
}
uct_ugni_ep_check_flush(desc->ep);
out:
/* have a go a processing the pending queue */
ucs_arbiter_dispatch(&iface->arbiter, 1, uct_ugni_ep_process_pending, NULL);
return;
}
void uct_ugni_progress(void *arg)
{
gni_cq_entry_t event_data = 0;
gni_post_descriptor_t *event_post_desc_ptr;
uct_ugni_base_desc_t *desc;
uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg;
gni_return_t ugni_rc;
ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data);
if (GNI_RC_NOT_DONE == ugni_rc) {
return;
}
if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) {
ucs_error("GNI_CqGetEvent falied. Error status %s %d ",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr);
if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) {
ucs_error("GNI_GetCompleted falied. Error status %s %d %d",
gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR);
return;
}
desc = (uct_ugni_base_desc_t *)event_post_desc_ptr;
ucs_trace_async("Completion received on %p", desc);
if (NULL != desc->comp_cb) {
uct_invoke_completion(desc->comp_cb);
}
--iface->outstanding;
--desc->ep->outstanding;
if (ucs_likely(desc->not_ready_to_free == 0)) {
ucs_mpool_put(desc);
}
return;
}
static UCS_F_ALWAYS_INLINE unsigned
uct_cuda_copy_progress_event_queue(ucs_queue_head_t *event_queue, unsigned max_events)
{
unsigned count = 0;
cudaError_t result = cudaSuccess;
uct_cuda_copy_event_desc_t *cuda_event;
ucs_queue_iter_t iter;
ucs_queue_for_each_safe(cuda_event, iter, event_queue, queue) {
result = cudaEventQuery(cuda_event->event);
if (cudaSuccess != result) {
break;
}
ucs_queue_del_iter(event_queue, iter);
if (cuda_event->comp != NULL) {
uct_invoke_completion(cuda_event->comp, UCS_OK);
}
ucs_trace_poll("CUDA Event Done :%p", cuda_event);
ucs_mpool_put(cuda_event);
count++;
if (count >= max_events) {
break;
}
}
/* Endpoint operations */
static inline void uct_ugni_invoke_orig_comp(uct_ugni_rdma_fetch_desc_t *fma, ucs_status_t status)
{
if (ucs_likely(NULL != fma->orig_comp_cb)) {
uct_invoke_completion(fma->orig_comp_cb, status);
}
}
static void uct_rc_verbs_ep_am_zcopy_handler(uct_rc_iface_send_op_t *op)
{
uct_rc_iface_send_desc_t *desc = ucs_derived_of(op, uct_rc_iface_send_desc_t);
uct_invoke_completion(desc->super.user_comp);
ucs_mpool_put(desc);
}
