ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
uint16_t exp_max_pi;
/*
* If we got completion for the last posted WQE, max_pi would be advanced
* to the value calculated from prev_sw_pi - which is the index where the last
* posted WQE started. See also uct_rc_mlx5_iface_poll_tx().
*/
exp_max_pi = uct_rc_mlx5_calc_max_pi(iface, ep->tx.prev_sw_pi);
if (ep->tx.max_pi == exp_max_pi) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
ucs_trace_data("ep %p is flushed", ep);
return UCS_OK;
}
if (ep->super.unsignaled != 0) {
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0);
if (status != UCS_OK) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return status;
}
}
return UCS_INPROGRESS;
}
ucs_status_t uct_dc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
ucs_status_t status;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) {
if (ep->dci != UCT_DC_MLX5_EP_NO_DCI) {
uct_rc_txqp_purge_outstanding(&iface->tx.dcis[ep->dci].txqp,
UCS_ERR_CANCELED, 0);
#if ENABLE_ASSERT
iface->tx.dcis[ep->dci].flags |= UCT_DC_DCI_FLAG_EP_CANCELED;
#endif
}
uct_ep_pending_purge(tl_ep, NULL, 0);
return UCS_OK;
}
if (!uct_rc_iface_has_tx_resources(&iface->super.super)) {
return UCS_ERR_NO_RESOURCE;
}
if (ep->dci == UCT_DC_MLX5_EP_NO_DCI) {
if (!uct_dc_mlx5_iface_dci_can_alloc(iface)) {
return UCS_ERR_NO_RESOURCE; /* waiting for dci */
} else {
UCT_TL_EP_STAT_FLUSH(&ep->super); /* no sends */
return UCS_OK;
}
}
if (!uct_dc_mlx5_iface_dci_ep_can_send(ep)) {
return UCS_ERR_NO_RESOURCE; /* cannot send */
}
status = uct_dc_mlx5_iface_flush_dci(iface, ep->dci);
if (status == UCS_OK) {
UCT_TL_EP_STAT_FLUSH(&ep->super);
return UCS_OK; /* all sends completed */
}
ucs_assert(status == UCS_INPROGRESS);
ucs_assert(ep->dci != UCT_DC_MLX5_EP_NO_DCI);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
return uct_rc_txqp_add_flush_comp(&iface->super.super, &ep->super, txqp,
comp, txwq->sig_pi);
}
ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
ucs_status_t status;
if (!uct_rc_iface_has_tx_resources(&iface->super)) {
return UCS_ERR_NO_RESOURCE;
}
if (uct_rc_txqp_available(&ep->super.txqp) == iface->config.tx_max_wr) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return UCS_OK;
}
if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) {
if (IBV_DEVICE_HAS_NOP(&uct_ib_iface_device(&iface->super.super)->dev_attr)) {
status = uct_rc_verbs_ep_nop(ep);
} else {
status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0);
}
if (status != UCS_OK) {
return status;
}
} else if (!uct_rc_ep_has_tx_resources(&ep->super)) {
return UCS_ERR_NO_RESOURCE;
}
uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi);
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uint16_t sn;
if (!uct_rc_iface_has_tx_resources(&iface->super)) {
return UCS_ERR_NO_RESOURCE;
}
if (uct_rc_txqp_available(&ep->super.txqp) == ep->tx.wq.bb_max) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return UCS_OK;
}
if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) {
sn = ep->tx.wq.sw_pi;
UCT_RC_CHECK_RES(&iface->super, &ep->super);
uct_rc_mlx5_txqp_inline_post(&iface->super, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
MLX5_OPCODE_NOP, NULL, 0,
0, 0,
0, 0,
NULL, 0);
} else if (!uct_rc_ep_has_tx_resources(&ep->super)) {
return UCS_ERR_NO_RESOURCE;
} else {
sn = ep->tx.wq.sig_pi;
}
uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, sn);
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
ucs_status_t uct_cm_ep_flush(uct_ep_h tl_ep)
{
ucs_status_t status;
status = uct_cm_iface_flush_do(tl_ep->iface);
if (status == UCS_OK) {
UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
} else {
UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t));
}
return status;
}
ucs_status_t uct_tcp_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_tcp_ep_t *ep = ucs_derived_of(tl_ep, uct_tcp_ep_t);
if (!uct_tcp_ep_can_send(ep)) {
return UCS_ERR_NO_RESOURCE;
}
ucs_assert(ucs_queue_is_empty(&ep->pending_q));
UCT_TL_EP_STAT_FLUSH(&ep->super);
return UCS_OK;
}
static ucs_status_t uct_ugni_smsg_ep_flush(uct_ep_h tl_ep){
uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t);
uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t);
if (0 == ep->super.outstanding) {
UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
return UCS_OK;
}
progress_local_cq(iface);
UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t));
return UCS_INPROGRESS;
}
ucs_status_t uct_dc_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp)
{
uct_dc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_iface_t);
uct_dc_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_ep_t);
ucs_status_t status;
/* If waiting for FC grant, return NO_RESOURCE to prevent ep destruction.
* Otherwise grant for destroyed ep will arrive and there will be a
* segfault when we will try to access the ep by address from the grant
* message. */
if (!uct_rc_iface_has_tx_resources(&iface->super)) {
return UCS_ERR_NO_RESOURCE;
}
if (ep->dci == UCT_DC_EP_NO_DCI) {
if (!uct_dc_iface_dci_can_alloc(iface)) {
return UCS_ERR_NO_RESOURCE; /* waiting for dci */
} else {
UCT_TL_EP_STAT_FLUSH(&ep->super); /* no sends */
return UCS_OK;
}
}
if (!uct_dc_iface_dci_ep_can_send(ep)) {
return UCS_ERR_NO_RESOURCE; /* cannot send */
}
status = uct_dc_iface_flush_dci(iface, ep->dci);
if (status == UCS_OK) {
UCT_TL_EP_STAT_FLUSH(&ep->super);
return UCS_OK; /* all sends completed */
}
ucs_assert(status == UCS_INPROGRESS);
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super);
return UCS_INPROGRESS;
}
ucs_status_t uct_mm_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_mm_ep_t *ep = ucs_derived_of(tl_ep, uct_mm_ep_t);
uct_mm_ep_update_cached_tail(ep);
if (!uct_mm_ep_has_tx_resources(ep)) {
return UCS_ERR_NO_RESOURCE;
}
ucs_memory_cpu_store_fence();
UCT_TL_EP_STAT_FLUSH(&ep->super);
return UCS_OK;
}
ucs_status_t uct_ugni_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t);
ucs_status_t status = UCS_INPROGRESS;
ucs_trace_func("tl_ep=%p, flags=%x, comp=%p", tl_ep, flags, comp);
if (uct_ugni_ep_can_flush(ep)) {
UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
return UCS_OK;
}
status = uct_ugni_add_flush_comp(ep, flags, comp);
if (UCS_INPROGRESS == status) {
UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t));
}
return status;
}
ucs_status_t uct_ugni_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t);
uct_ugni_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ugni_iface_t);
if (uct_ugni_can_flush(ep)) {
UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
return UCS_OK;
}
ep->flush_flag = 1;
ep->arb_flush = ep->arb_size;
uct_ugni_progress(iface);
UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t));
return UCS_ERR_NO_RESOURCE;
}
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
if (ep->super.available == ep->tx.wq.bb_max) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
ucs_trace_data("ep %p is flushed", ep);
return UCS_OK;
}
if (ep->super.unsignaled != 0) {
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0);
if (status != UCS_OK) {
return status;
}
}
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
static ucs_status_t uct_ugni_smsg_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t);
ucs_status_t status;
/* Even if we have no outstanding requests we can still get back send credits */
status = progress_local_cq(ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t));
if((0 == ep->super.outstanding) && (ep->super.arb_size == 0)) {
ep->super.flush_flag = 0;
}
if(uct_ugni_can_flush(&ep->super)) {
UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
status = UCS_OK;
} else {
ep->super.flush_flag = 1;
UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t));
status = UCS_ERR_NO_RESOURCE;
}
return status;
}
ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
ucs_status_t status;
if (ep->tx.available == iface->super.config.tx_qp_len) {
UCT_TL_EP_STAT_FLUSH(&ep->super);
return UCS_OK;
}
if (ep->super.unsignaled != 0) {
if (IBV_DEVICE_HAS_NOP(&uct_ib_iface_device(&iface->super.super)->dev_attr)) {
status = uct_rc_verbs_ep_nop(ep);
} else {
status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0);
}
if (status != UCS_OK) {
return status;
}
}
return UCS_INPROGRESS;
}