static ucs_status_t uct_ugni_add_flush_comp(uct_ugni_ep_t *ep, unsigned flags, uct_completion_t *comp) { uct_ugni_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_ugni_iface_t); uct_ugni_flush_group_t *new_group, *present_group; if (!uct_ugni_ep_can_send(ep)) { return UCS_ERR_NO_RESOURCE; } if (NULL == comp) { return UCS_INPROGRESS; } new_group = uct_ugni_new_flush_group(iface); new_group->flush_comp.count = UCT_UGNI_INIT_FLUSH_REQ; #ifdef DEBUG new_group->flush_comp.func = NULL; new_group->parent = NULL; #endif present_group = (uct_ugni_flush_group_t*)uct_ugni_safe_swap_pointers(&ep->flush_group, (uintptr_t)new_group); present_group->flush_comp.func = uct_ugni_flush_cb; present_group->user_comp = comp; present_group->parent = new_group; uct_invoke_completion(&present_group->flush_comp, UCS_OK); return UCS_INPROGRESS; }
static void uct_ugni_flush_cb(uct_completion_t *self, ucs_status_t status) { uct_ugni_flush_group_t *group = ucs_container_of(self, uct_ugni_flush_group_t, flush_comp); ucs_trace("group=%p, parent=%p, user_comp=%p", group, group->parent, group->user_comp); uct_invoke_completion(group->user_comp, UCS_OK); uct_ugni_check_flush(group->parent); uct_ugni_put_flush_group(group); }
void uct_ugni_progress(void *arg) { gni_cq_entry_t event_data = 0; gni_post_descriptor_t *event_post_desc_ptr; uct_ugni_base_desc_t *desc; uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg; gni_return_t ugni_rc; ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data); if (GNI_RC_NOT_DONE == ugni_rc) { goto out; } if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) { ucs_error("GNI_CqGetEvent falied. Error status %s %d ", gni_err_str[ugni_rc], ugni_rc); return; } ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr); if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) { ucs_error("GNI_GetCompleted falied. Error status %s %d %d", gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR); return; } desc = (uct_ugni_base_desc_t *)event_post_desc_ptr; ucs_trace_async("Completion received on %p", desc); if (NULL != desc->comp_cb) { uct_invoke_completion(desc->comp_cb, UCS_OK); } --iface->outstanding; --desc->ep->outstanding; if (ucs_likely(0 == desc->not_ready_to_free)) { ucs_mpool_put(desc); } uct_ugni_ep_check_flush(desc->ep); out: /* have a go a processing the pending queue */ ucs_arbiter_dispatch(&iface->arbiter, 1, uct_ugni_ep_process_pending, NULL); return; }
void uct_ugni_progress(void *arg) { gni_cq_entry_t event_data = 0; gni_post_descriptor_t *event_post_desc_ptr; uct_ugni_base_desc_t *desc; uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg; gni_return_t ugni_rc; ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data); if (GNI_RC_NOT_DONE == ugni_rc) { return; } if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) { ucs_error("GNI_CqGetEvent falied. Error status %s %d ", gni_err_str[ugni_rc], ugni_rc); return; } ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr); if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) { ucs_error("GNI_GetCompleted falied. Error status %s %d %d", gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR); return; } desc = (uct_ugni_base_desc_t *)event_post_desc_ptr; ucs_trace_async("Completion received on %p", desc); if (NULL != desc->comp_cb) { uct_invoke_completion(desc->comp_cb); } --iface->outstanding; --desc->ep->outstanding; if (ucs_likely(desc->not_ready_to_free == 0)) { ucs_mpool_put(desc); } return; }
static UCS_F_ALWAYS_INLINE unsigned uct_cuda_copy_progress_event_queue(ucs_queue_head_t *event_queue, unsigned max_events) { unsigned count = 0; cudaError_t result = cudaSuccess; uct_cuda_copy_event_desc_t *cuda_event; ucs_queue_iter_t iter; ucs_queue_for_each_safe(cuda_event, iter, event_queue, queue) { result = cudaEventQuery(cuda_event->event); if (cudaSuccess != result) { break; } ucs_queue_del_iter(event_queue, iter); if (cuda_event->comp != NULL) { uct_invoke_completion(cuda_event->comp, UCS_OK); } ucs_trace_poll("CUDA Event Done :%p", cuda_event); ucs_mpool_put(cuda_event); count++; if (count >= max_events) { break; } }
/* Endpoint operations */ static inline void uct_ugni_invoke_orig_comp(uct_ugni_rdma_fetch_desc_t *fma, ucs_status_t status) { if (ucs_likely(NULL != fma->orig_comp_cb)) { uct_invoke_completion(fma->orig_comp_cb, status); } }
static void uct_rc_verbs_ep_am_zcopy_handler(uct_rc_iface_send_op_t *op) { uct_rc_iface_send_desc_t *desc = ucs_derived_of(op, uct_rc_iface_send_desc_t); uct_invoke_completion(desc->super.user_comp); ucs_mpool_put(desc); }