static inline ucs_status_t uct_mm_ep_get_remote_elem(uct_mm_ep_t *ep, uint64_t head,
uct_mm_fifo_element_t **elem)
{
uct_mm_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_mm_iface_t);
uint64_t elem_index; /* the fifo elem's index in the fifo. */
/* must be smaller than fifo size */
uint64_t returned_val;
elem_index = ep->fifo_ctl->head & iface->fifo_mask;
*elem = UCT_MM_IFACE_GET_FIFO_ELEM(iface, ep->fifo, elem_index);
/* try to get ownership of the head element */
returned_val = ucs_atomic_cswap64(&ep->fifo_ctl->head, head, head+1);
if (returned_val != head) {
return UCS_ERR_NO_RESOURCE;
}
return UCS_OK;
}
static inline void uct_mm_iface_poll_fifo(uct_mm_iface_t *iface)
{
uint64_t read_index_loc, read_index;
uct_mm_fifo_element_t* read_index_elem;
ucs_status_t status;
/* check the memory pool to make sure that there is a new descriptor available */
if (ucs_unlikely(iface->last_recv_desc == NULL)) {
UCT_TL_IFACE_GET_RX_DESC(&iface->super, &iface->recv_desc_mp,
iface->last_recv_desc, return);
}
read_index = iface->read_index;
read_index_loc = (read_index & iface->fifo_mask);
/* the fifo_element which the read_index points to */
read_index_elem = UCT_MM_IFACE_GET_FIFO_ELEM(iface, iface->recv_fifo_elements ,read_index_loc);
/* check the read_index to see if there is a new item to read (checking the owner bit) */
if (((read_index >> iface->fifo_shift) & 1) == ((read_index_elem->flags) & 1)) {
/* read from read_index_elem */
ucs_memory_cpu_load_fence();
ucs_assert(iface->read_index <= iface->recv_fifo_ctl->head);
status = uct_mm_iface_process_recv(iface, read_index_elem);
if (status != UCS_OK) {
/* the last_recv_desc is in use. get a new descriptor for it */
UCT_TL_IFACE_GET_RX_DESC(&iface->super, &iface->recv_desc_mp,
iface->last_recv_desc, ucs_debug("recv mpool is empty"));
}
