static void rxd_peer_timeout(struct rxd_ep *rxd_ep, struct rxd_peer *peer)
{
struct fi_cq_err_entry err_entry;
struct rxd_x_entry *tx_entry;
struct rxd_pkt_entry *pkt_entry;
int ret;
while (!dlist_empty(&peer->tx_list)) {
dlist_pop_front(&peer->tx_list, struct rxd_x_entry, tx_entry, entry);
memset(&err_entry, 0, sizeof(struct fi_cq_err_entry));
rxd_tx_entry_free(rxd_ep, tx_entry);
err_entry.op_context = tx_entry->cq_entry.op_context;
err_entry.flags = tx_entry->cq_entry.flags;
err_entry.err = FI_ECONNREFUSED;
err_entry.prov_errno = 0;
ret = ofi_cq_write_error(&rxd_ep_tx_cq(rxd_ep)->util_cq, &err_entry);
if (ret)
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not write error entry\n");
}
while (!dlist_empty(&peer->unacked)) {
dlist_pop_front(&peer->unacked, struct rxd_pkt_entry, pkt_entry,
d_entry);
ofi_buf_free(pkt_entry);
peer->unacked_cnt--;
}
dlist_remove(&peer->entry);
}
static void rxd_close_peer(struct rxd_ep *ep, struct rxd_peer *peer)
{
struct rxd_pkt_entry *pkt_entry;
struct rxd_x_entry *x_entry;
while (!dlist_empty(&peer->unacked)) {
dlist_pop_front(&peer->unacked, struct rxd_pkt_entry,
pkt_entry, d_entry);
ofi_buf_free(pkt_entry);
peer->unacked_cnt--;
}
while(!dlist_empty(&peer->tx_list)) {
dlist_pop_front(&peer->tx_list, struct rxd_x_entry,
x_entry, entry);
rxd_tx_entry_free(ep, x_entry);
}
while(!dlist_empty(&peer->rx_list)) {
dlist_pop_front(&peer->rx_list, struct rxd_x_entry,
x_entry, entry);
rxd_rx_entry_free(ep, x_entry);
}
while(!dlist_empty(&peer->rma_rx_list)) {
dlist_pop_front(&peer->rma_rx_list, struct rxd_x_entry,
x_entry, entry);
rxd_tx_entry_free(ep, x_entry);
}
dlist_remove(&peer->entry);
peer->active = 0;
}
static void rxd_progress_pkt_list(struct rxd_ep *ep, struct rxd_peer *peer)
{
struct rxd_pkt_entry *pkt_entry;
uint64_t current;
int ret, retry = 0;
current = fi_gettime_ms();
if (peer->retry_cnt > RXD_MAX_PKT_RETRY) {
rxd_peer_timeout(ep, peer);
return;
}
dlist_foreach_container(&peer->unacked, struct rxd_pkt_entry,
pkt_entry, d_entry) {
if (pkt_entry->flags & (RXD_PKT_IN_USE | RXD_PKT_ACKED) ||
current < rxd_get_retry_time(pkt_entry->timestamp, peer->retry_cnt))
continue;
retry = 1;
ret = rxd_ep_send_pkt(ep, pkt_entry);
if (ret)
break;
}
if (retry)
peer->retry_cnt++;
if (!dlist_empty(&peer->unacked))
ep->next_retry = ep->next_retry == -1 ? peer->retry_cnt :
MIN(ep->next_retry, peer->retry_cnt);
}
int rxm_ep_recv_common(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, uint64_t tag, uint64_t ignore,
void *context, uint64_t flags, int op)
{
struct rxm_recv_entry *recv_entry;
struct rxm_ep *rxm_ep;
struct rxm_recv_queue *recv_queue;
dlist_func_t *match;
int ret, i;
rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
// TODO pass recv_queue as arg
if (op == ofi_op_msg) {
recv_queue = &rxm_ep->recv_queue;
match = ofi_match_unexp_msg;
} else if (op == ofi_op_tagged) {
recv_queue = &rxm_ep->trecv_queue;
match = ofi_match_unexp_msg_tagged;
} else {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Unknown op!\n");
return -FI_EINVAL;
}
if (freestack_isempty(recv_queue->recv_fs)) {
FI_DBG(&rxm_prov, FI_LOG_CQ, "Exhaused recv_entry freestack\n");
return -FI_EAGAIN;
}
recv_entry = freestack_pop(recv_queue->recv_fs);
for (i = 0; i < count; i++) {
recv_entry->iov[i].iov_base = iov[i].iov_base;
recv_entry->iov[i].iov_len = iov[i].iov_len;
recv_entry->desc[i] = desc[i];
FI_DBG(&rxm_prov, FI_LOG_EP_CTRL, "post recv: %u\n",
iov[i].iov_len);
}
recv_entry->count = count;
recv_entry->addr = (rxm_ep->rxm_info->caps & FI_DIRECTED_RECV) ?
src_addr : FI_ADDR_UNSPEC;
recv_entry->flags = flags;
if (op == ofi_op_tagged) {
recv_entry->tag = tag;
recv_entry->ignore = ignore;
}
if (!dlist_empty(&recv_queue->unexp_msg_list)) {
ret = rxm_check_unexp_msg_list(rxm_ep->util_ep.rx_cq, recv_queue,
recv_entry, match);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"Unable to check unexp msg list\n");
return ret;
}
}
dlist_insert_tail(&recv_entry->entry, &recv_queue->recv_list);
return 0;
}
int _gnix_dgram_alloc(struct gnix_dgram_hndl *hndl, enum gnix_dgram_type type,
struct gnix_datagram **d_ptr)
{
int ret = -FI_EAGAIN;
struct gnix_datagram *d = NULL;
struct dlist_entry *the_free_list;
struct dlist_entry *the_active_list;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
fastlock_acquire(&hndl->lock);
if (type == GNIX_DGRAM_WC) {
the_free_list = &hndl->wc_dgram_free_list;
the_active_list = &hndl->wc_dgram_active_list;
} else {
the_free_list = &hndl->bnd_dgram_free_list;
the_active_list = &hndl->bnd_dgram_active_list;
}
if (!dlist_empty(the_free_list)) {
d = dlist_first_entry(the_free_list, struct gnix_datagram,
list);
if (d != NULL) {
dlist_remove_init(&d->list);
dlist_insert_head(&d->list, the_active_list);
d->type = type;
ret = FI_SUCCESS;
}
}
ssize_t rxd_send_rts_if_needed(struct rxd_ep *ep, fi_addr_t addr)
{
if (ep->peers[addr].peer_addr == FI_ADDR_UNSPEC &&
dlist_empty(&ep->peers[addr].unacked))
return rxd_ep_send_rts(ep, addr);
return 0;
}
void slip_reset_parser(pSlip slip)
{
dlist_empty(slip->parse_data.lstTokens);
slip->parse_data.current_line = 0;
slip->parse_data.eCurrentToken = NULL;
slip->parse_data.comment_depth = 0;
}
static int util_wait_fd_close(struct fid *fid)
{
struct util_wait_fd *wait;
struct ofi_wait_fd_entry *fd_entry;
int ret;
wait = container_of(fid, struct util_wait_fd, util_wait.wait_fid.fid);
ret = fi_wait_cleanup(&wait->util_wait);
if (ret)
return ret;
fastlock_acquire(&wait->lock);
while (!dlist_empty(&wait->fd_list)) {
dlist_pop_front(&wait->fd_list, struct ofi_wait_fd_entry,
fd_entry, entry);
fi_epoll_del(wait->epoll_fd, fd_entry->fd);
free(fd_entry);
}
fastlock_release(&wait->lock);
fi_epoll_del(wait->epoll_fd, wait->signal.fd[FI_READ_FD]);
fd_signal_free(&wait->signal);
fi_epoll_close(wait->epoll_fd);
fastlock_destroy(&wait->lock);
free(wait);
return 0;
}
static void fi_ibv_verbs_devs_free(struct dlist_entry *verbs_devs)
{
struct verbs_dev_info *dev;
struct verbs_addr *addr;
while (!dlist_empty(verbs_devs)) {
dlist_pop_front(verbs_devs, struct verbs_dev_info, dev, entry);
while (!dlist_empty(&dev->addrs)) {
dlist_pop_front(&dev->addrs, struct verbs_addr, addr, entry);
rdma_freeaddrinfo(addr->rai);
free(addr);
}
free(dev->name);
free(dev);
}
}
//push消息并执行同步操作
static inline void msgque_sync_push(ptq_t ptq)
{
assert(ptq->mode == MSGQ_WRITE);
if(ptq->mode != MSGQ_WRITE) return;
msgque_t que = ptq->que;
mutex_lock(que->mtx);
uint8_t empty = llist_is_empty(&que->share_que);
llist_swap(&que->share_que,&ptq->local_que);
if(empty){
struct dnode *l = dlist_pop(&que->blocks);
if(l){
//if there is a block per_thread_struct wake it up
ptq_t block_ptq = (ptq_t)l;
mutex_unlock(que->mtx);
condition_signal(block_ptq->cond);
}
}
//对所有在can_interrupt中的元素调用回调
while(!dlist_empty(&que->can_interrupt))
{
ptq_t ptq = (ptq_t)dlist_pop(&que->can_interrupt);
ptq->read_que.notify_function(ptq->read_que.ud);
}
mutex_unlock(que->mtx);
}
void rxd_rx_entry_release(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry)
{
rx_entry->key = -1;
dlist_remove(&rx_entry->entry);
freestack_push(ep->rx_entry_fs, rx_entry);
if (ep->credits && !dlist_empty(&ep->wait_rx_list))
rxd_check_waiting_rx(ep);
}
static void __gnix_vc_cancel(struct gnix_vc *vc)
{
struct gnix_nic *nic = vc->ep->nic;
fastlock_acquire(&nic->rx_vc_lock);
if (!dlist_empty(&vc->rx_list))
dlist_remove(&vc->rx_list);
fastlock_release(&nic->rx_vc_lock);
fastlock_acquire(&nic->work_vc_lock);
if (!dlist_empty(&vc->work_list))
dlist_remove(&vc->work_list);
fastlock_release(&nic->work_vc_lock);
fastlock_acquire(&nic->tx_vc_lock);
if (!dlist_empty(&vc->tx_list))
dlist_remove(&vc->tx_list);
fastlock_release(&nic->tx_vc_lock);
}
struct sock_domain *sock_dom_list_head(void)
{
struct sock_domain *domain;
fastlock_acquire(&sock_list_lock);
if (dlist_empty(&sock_dom_list)) {
domain = NULL;
} else {
domain = container_of(sock_dom_list.next,
struct sock_domain, dom_list_entry);
}
fastlock_release(&sock_list_lock);
return domain;
}
struct sock_fabric *sock_fab_list_head(void)
{
struct sock_fabric *fabric;
fastlock_acquire(&sock_list_lock);
if (dlist_empty(&sock_fab_list)) {
fabric = NULL;
} else {
fabric = container_of(sock_fab_list.next,
struct sock_fabric, fab_list_entry);
}
fastlock_release(&sock_list_lock);
return fabric;
}
static void process_pe_lists(struct tcpx_ep *ep)
{
struct tcpx_pe_entry *pe_entry;
struct dlist_entry *entry;
if (dlist_empty(&ep->rx_queue))
goto tx_pe_list;
entry = ep->rx_queue.next;
pe_entry = container_of(entry, struct tcpx_pe_entry,
entry);
process_rx_pe_entry(pe_entry);
tx_pe_list:
if (dlist_empty(&ep->tx_queue))
return ;
entry = ep->tx_queue.next;
pe_entry = container_of(entry, struct tcpx_pe_entry,
entry);
process_tx_pe_entry(pe_entry);
}
struct ofi_subscription *ofi_monitor_get_event(struct ofi_notification_queue *nq)
{
struct ofi_subscription *subscription;
util_monitor_read_events(nq->monitor);
fastlock_acquire(&nq->lock);
if (!dlist_empty(&nq->list)) {
dlist_pop_front(&nq->list, struct ofi_subscription,
subscription, entry);
/* needed to protect against double insertions */
dlist_init(&subscription->entry);
} else {
static void rxd_ep_progress(struct util_ep *util_ep)
{
struct rxd_peer *peer;
struct fi_cq_msg_entry cq_entry;
struct dlist_entry *tmp;
struct rxd_ep *ep;
ssize_t ret;
int i;
ep = container_of(util_ep, struct rxd_ep, util_ep);
fastlock_acquire(&ep->util_ep.lock);
for(ret = 1, i = 0;
ret > 0 && (!rxd_env.spin_count || i < rxd_env.spin_count);
i++) {
ret = fi_cq_read(ep->dg_cq, &cq_entry, 1);
if (ret == -FI_EAGAIN)
break;
if (ret == -FI_EAVAIL) {
rxd_handle_error(ep);
continue;
}
if (cq_entry.flags & FI_RECV)
rxd_handle_recv_comp(ep, &cq_entry);
else
rxd_handle_send_comp(ep, &cq_entry);
}
if (!rxd_env.retry)
goto out;
ep->next_retry = -1;
dlist_foreach_container_safe(&ep->rts_sent_list, struct rxd_peer,
peer, entry, tmp)
rxd_progress_pkt_list(ep, peer);
dlist_foreach_container_safe(&ep->active_peers, struct rxd_peer,
peer, entry, tmp) {
rxd_progress_pkt_list(ep, peer);
if (dlist_empty(&peer->unacked))
rxd_progress_tx_list(ep, peer);
}
out:
while (ep->posted_bufs < ep->rx_size && !ret)
ret = rxd_ep_post_buf(ep);
fastlock_release(&ep->util_ep.lock);
}
void rxd_tx_entry_done(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
{
struct rxd_pkt_meta *pkt_meta;
struct dlist_entry *item;
while (!dlist_empty(&tx_entry->pkt_list)) {
item = tx_entry->pkt_list.next;
pkt_meta = container_of(item, struct rxd_pkt_meta, entry);
dlist_remove(&pkt_meta->entry);
RXD_PKT_MARK_REMOTE_ACK(pkt_meta);
rxd_tx_pkt_release(pkt_meta);
}
rxd_tx_entry_release(ep, tx_entry);
}
/**
* Find the first free block in list i.
*
* @return 1 if the block cannot be found.
*
* @return 0 if the block is found.
*/
static inline int __gnix_buddy_find_block(gnix_buddy_alloc_handle_t
*alloc_handle, uint32_t i, void **ptr)
{
uint32_t j;
for (j = i; j < alloc_handle->nlists; j++) {
if (!dlist_empty(alloc_handle->lists + j)) {
__gnix_buddy_split(alloc_handle, j, i, ptr);
return 0;
}
}
return 1;
}
static void rxd_check_waiting_rx(struct rxd_ep *ep)
{
struct dlist_entry *entry;
struct rxd_rx_entry *rx_entry;
if (!ep->credits)
return;
while(!dlist_empty(&ep->wait_rx_list) && ep->credits) {
entry = ep->wait_rx_list.next;
rx_entry = container_of(entry, struct rxd_rx_entry, wait_entry);
rxd_progress_wait_rx(ep, rx_entry);
}
}
void ofi_monitor_del_cache(struct ofi_mr_cache *cache)
{
struct ofi_mem_monitor *monitor = cache->monitor;
if (!monitor)
return;
fastlock_acquire(&monitor->lock);
dlist_remove(&cache->notify_entry);
if (dlist_empty(&monitor->list) && (monitor == uffd_monitor))
ofi_uffd_cleanup();
fastlock_release(&monitor->lock);
}
void rxd_tx_entry_done(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
{
struct rxd_pkt_meta *pkt_meta;
while (!dlist_empty(&tx_entry->pkt_list)) {
pkt_meta = container_of(tx_entry->pkt_list.next,
struct rxd_pkt_meta, entry);
dlist_remove(&pkt_meta->entry);
if (pkt_meta->flags & RXD_LOCAL_COMP)
rxd_tx_pkt_free(pkt_meta);
else
pkt_meta->flags |= RXD_REMOTE_ACK;
}
rxd_tx_entry_free(ep, tx_entry);
}
static void free_i18n_strings(void)
{
struct dlist_head * entry;
i18nstr_t * istr;
while (!dlist_empty(&g_i18n_strings))
{
entry = g_i18n_strings.next;
dlist_del(entry);
istr = dlist_entry(entry, i18nstr_t, list);
if (istr->string) FREE(istr->string);
FREE(istr);
}
}
bool ofi_mr_cache_flush(struct ofi_mr_cache *cache)
{
struct ofi_mr_entry *entry;
if (dlist_empty(&cache->lru_list))
return false;
dlist_pop_front(&cache->lru_list, struct ofi_mr_entry,
entry, lru_entry);
dlist_init(&entry->lru_entry);
FI_DBG(cache->domain->prov, FI_LOG_MR, "flush %p (len: %" PRIu64 ")\n",
entry->iov.iov_base, entry->iov.iov_len);
util_mr_uncache_entry(cache, entry);
util_mr_free_entry(cache, entry);
return true;
}
static inline void sock_cq_copy_overflow_list(struct sock_cq *cq, size_t count)
{
ssize_t i;
struct sock_cq_overflow_entry_t *overflow_entry;
for (i = 0; i < count && !dlist_empty(&cq->overflow_list); i++) {
overflow_entry = container_of(cq->overflow_list.next,
struct sock_cq_overflow_entry_t,
entry);
rbwrite(&cq->addr_rb, &overflow_entry->addr, sizeof(fi_addr_t));
rbcommit(&cq->addr_rb);
rbfdwrite(&cq->cq_rbfd, &overflow_entry->cq_entry[0], overflow_entry->len);
rbfdcommit(&cq->cq_rbfd);
dlist_remove(&overflow_entry->entry);
free(overflow_entry);
}
}
static int fi_ibv_get_srcaddr_devs(struct fi_info **info)
{
struct fi_info *fi, *add_info;
struct fi_info *fi_unconf = NULL, *fi_prev = NULL;
struct verbs_dev_info *dev;
struct verbs_addr *addr;
int ret = 0;
DEFINE_LIST(verbs_devs);
ret = fi_ibv_getifaddrs(&verbs_devs);
if (ret)
return ret;
if (dlist_empty(&verbs_devs)) {
VERBS_WARN(FI_LOG_CORE, "No interface address found\n");
return 0;
}
for (fi = *info; fi; fi = fi->next) {
dlist_foreach_container(&verbs_devs, struct verbs_dev_info, dev, entry)
if (!strncmp(fi->domain_attr->name, dev->name, strlen(dev->name))) {
dlist_foreach_container(&dev->addrs, struct verbs_addr, addr, entry) {
/* When a device has multiple interfaces/addresses configured
* duplicate fi_info and add the address info. fi->src_addr
* would have been set in the previous iteration */
if (fi->src_addr) {
if (!(add_info = fi_dupinfo(fi))) {
ret = -FI_ENOMEM;
goto out;
}
add_info->next = fi->next;
fi->next = add_info;
fi = add_info;
}
ret = fi_ibv_rai_to_fi(addr->rai, fi);
if (ret)
goto out;
}
break;
}
}
static int task_init()
{
pthread_t tid;
rec_task_list = dlist_init();
rmd_devmgr_travel_channel(add_task, NULL);
if (dlist_empty(rec_task_list)) /* have none record task */
return -1;
sl = single_login_init();
if (sl == NULL)
return -1;
if (pthread_create(&tid, NULL, task_scheduler, NULL) != 0)
return -1;
pthread_detach(tid);
return 0;
}
static inline gnix_ht_entry_t *__gnix_ht_lookup_entry(
struct dlist_entry *head,
gnix_ht_key_t key,
uint64_t *collision_count)
{
gnix_ht_entry_t *ht_entry;
if (dlist_empty(head))
return NULL;
dlist_for_each(head, ht_entry, entry) {
if (ht_entry->key == key)
return ht_entry;
if (collision_count)
*collision_count += 1;
}
return NULL;
}
static int sock_poll_close(fid_t fid)
{
struct sock_poll *poll;
struct sock_fid_list *list_item;
struct dlist_entry *p, *head;
poll = container_of(fid, struct sock_poll, poll_fid.fid);
head = &poll->fid_list;
while (!dlist_empty(head)) {
p = head->next;
list_item = container_of(p, struct sock_fid_list, entry);
dlist_remove(p);
free(list_item);
}
atomic_dec(&poll->domain->ref);
free(poll);
return 0;
}
int thrd_start (thrd_pool_t *pool)
{
diter_t *i;
int err;
struct timespec now;
if (dlist_empty(pool->threads)) {
pool->status |= THRD_ERR_EMPTY;
return -1;
}
/* We need to bulid the priority set the first time.
*
* Currently this "first time" check is useless, since for the moment
* this structure is not supporting thread stopping.
*/
if ((pool->status & THRD_POOL_SORTED) == 0) {
if (set_rm_priorities(&pool->threads, pool->minprio) == -1) {
pool->status |= THRD_ERR_NULLPER;
return -1;
}
}
/* Start each thread. */
pool->status |= THRD_POOL_ACTIVE;
i = dlist_iter_new(&pool->threads);
rtutils_get_now(&now);
while (diter_hasnext(i)) {
err = startup((thrd_t *) diter_next(i), &now);
if (err) {
pool->status |= THRD_ERR_LIBRARY;
pool->err = err;
dlist_iter_free(i);
return -1;
}
}
dlist_iter_free(i);
return 0;
}