void uct_ugni_udt_progress(void *arg)
{
uct_ugni_udt_iface_t * iface = (uct_ugni_udt_iface_t *)arg;
uct_ugni_enter_async(&iface->super);
ucs_arbiter_dispatch(&iface->super.arbiter, 1, uct_ugni_udt_ep_process_pending, NULL);
uct_ugni_leave_async(&iface->super);
}
static void uct_ugni_smsg_progress(void *arg)
{
uct_ugni_smsg_iface_t *iface = (uct_ugni_smsg_iface_t *)arg;
progress_local_cq(iface);
progress_remote_cq(iface);
/* have a go a processing the pending queue */
ucs_arbiter_dispatch(&iface->super.arbiter, 1, uct_ugni_ep_process_pending, NULL);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_poll_tx(uct_dc_mlx5_iface_t *iface)
{
uint8_t dci;
struct mlx5_cqe64 *cqe;
uint32_t qp_num;
uint16_t hw_ci;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
cqe = uct_ib_mlx5_poll_cq(&iface->super.super.super, &iface->mlx5_common.tx.cq);
if (cqe == NULL) {
return;
}
UCS_STATS_UPDATE_COUNTER(iface->super.super.stats, UCT_RC_IFACE_STAT_TX_COMPLETION, 1);
ucs_memory_cpu_load_fence();
qp_num = ntohl(cqe->sop_drop_qpn) & UCS_MASK(UCT_IB_QPN_ORDER);
dci = uct_dc_iface_dci_find(&iface->super, qp_num);
txqp = &iface->super.tx.dcis[dci].txqp;
txwq = &iface->dci_wqs[dci];
hw_ci = ntohs(cqe->wqe_counter);
ucs_trace_poll("dc_mlx5 iface %p tx_cqe: dci[%d] qpn 0x%x txqp %p hw_ci %d",
iface, dci, qp_num, txqp, hw_ci);
uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci));
uct_dc_iface_dci_put(&iface->super, dci);
uct_rc_mlx5_txqp_process_tx_cqe(txqp, cqe, hw_ci);
iface->super.super.tx.cq_available++;
if (uct_dc_iface_dci_can_alloc(&iface->super)) {
ucs_arbiter_dispatch(uct_dc_iface_dci_waitq(&iface->super), 1,
uct_dc_iface_dci_do_pending_wait, NULL);
}
ucs_arbiter_dispatch(uct_dc_iface_tx_waitq(&iface->super), 1,
uct_dc_iface_dci_do_pending_tx, NULL);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_poll_tx(uct_dc_mlx5_iface_t *iface)
{
uint8_t dci;
struct mlx5_cqe64 *cqe;
uint32_t qp_num;
uint16_t hw_ci;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
cqe = uct_ib_mlx5_get_cqe(&iface->super.super.super, &iface->mlx5_common.tx.cq,
iface->mlx5_common.tx.cq.cqe_size_log);
if (cqe == NULL) {
return;
}
UCS_STATS_UPDATE_COUNTER(iface->super.super.stats, UCT_RC_IFACE_STAT_TX_COMPLETION, 1);
ucs_memory_cpu_load_fence();
ucs_assertv(!(cqe->op_own & (MLX5_INLINE_SCATTER_32|MLX5_INLINE_SCATTER_64)),
"tx inline scatter not supported");
qp_num = ntohl(cqe->sop_drop_qpn) & UCS_MASK(UCT_IB_QPN_ORDER);
dci = uct_dc_iface_dci_find(&iface->super, qp_num);
txqp = &iface->super.tx.dcis[dci].txqp;
txwq = &iface->dci_wqs[dci];
hw_ci = ntohs(cqe->wqe_counter);
uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci));
uct_rc_txqp_completion(txqp, hw_ci);
iface->super.super.tx.cq_available++;
uct_dc_iface_dci_put(&iface->super, dci);
if (uct_dc_iface_dci_can_alloc(&iface->super)) {
ucs_arbiter_dispatch(&iface->super.super.tx.arbiter, 1, uct_dc_iface_dci_do_pending_wait, NULL);
}
ucs_arbiter_dispatch(&iface->super.tx.dci_arbiter, 1, uct_dc_iface_dci_do_pending_tx, NULL);
}
static void uct_ugni_smsg_progress(void *arg)
{
uct_ugni_smsg_iface_t *iface = (uct_ugni_smsg_iface_t *)arg;
ucs_status_t status;
do {
status = progress_local_cq(iface);
} while(status == UCS_INPROGRESS);
do {
status = progress_remote_cq(iface);
} while(status == UCS_INPROGRESS);
/* have a go a processing the pending queue */
ucs_arbiter_dispatch(&iface->super.arbiter, iface->config.smsg_max_credit,
uct_ugni_ep_process_pending, NULL);
}
void uct_ugni_progress(void *arg)
{
gni_cq_entry_t event_data = 0;
gni_post_descriptor_t *event_post_desc_ptr;
uct_ugni_base_desc_t *desc;
uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg;
gni_return_t ugni_rc;
ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data);
if (GNI_RC_NOT_DONE == ugni_rc) {
goto out;
}
if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) {
ucs_error("GNI_CqGetEvent falied. Error status %s %d ",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr);
if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) {
ucs_error("GNI_GetCompleted falied. Error status %s %d %d",
gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR);
return;
}
desc = (uct_ugni_base_desc_t *)event_post_desc_ptr;
ucs_trace_async("Completion received on %p", desc);
if (NULL != desc->comp_cb) {
uct_invoke_completion(desc->comp_cb, UCS_OK);
}
--iface->outstanding;
--desc->ep->outstanding;
if (ucs_likely(0 == desc->not_ready_to_free)) {
ucs_mpool_put(desc);
}
uct_ugni_ep_check_flush(desc->ep);
out:
/* have a go a processing the pending queue */
ucs_arbiter_dispatch(&iface->arbiter, 1, uct_ugni_ep_process_pending, NULL);
return;
}
