int _gnix_cm_nic_progress(struct gnix_cm_nic *cm_nic)
{
int ret = FI_SUCCESS;
int complete;
struct gnix_work_req *p = NULL;
/*
* if we're doing FI_PROGRESS_MANUAL,
* see what's going on inside kgni's datagram
* box...
*/
if (cm_nic->control_progress == FI_PROGRESS_MANUAL) {
ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
GNIX_DGRAM_NOBLOCK);
if (ret != FI_SUCCESS)
goto err;
}
/*
* do a quick check if queue doesn't have anything yet,
* don't need this to be atomic
*/
check_again:
if (list_empty(&cm_nic->cm_nic_wq))
return ret;
/*
* okay, stuff to do, lock work queue,
* dequeue head, unlock, process work element,
* if it doesn't compete, put back at the tail
* of the queue.
*/
fastlock_acquire(&cm_nic->wq_lock);
p = list_top(&cm_nic->cm_nic_wq, struct gnix_work_req, list);
if (p == NULL) {
fastlock_release(&cm_nic->wq_lock);
return ret;
}
gnix_list_del_init(&p->list);
fastlock_release(&cm_nic->wq_lock);
assert(p->progress_fn);
ret = p->progress_fn(p->data, &complete);
if (ret != FI_SUCCESS) {
/*
* TODO: fix this
*/
}
if (complete == 1) {
if (p->completer_fn) {
ret = p->completer_fn(p->completer_data);
free(p);
if (ret != FI_SUCCESS)
goto err;
}
goto check_again;
} else {
fastlock_acquire(&cm_nic->wq_lock);
list_add_tail(&cm_nic->cm_nic_wq, &p->list);
fastlock_release(&cm_nic->wq_lock);
}
err:
return ret;
}
/*
* this function is intended to be invoked as an argument to pthread_create,
*/
static void *_gnix_dgram_prog_thread_fn(void *the_arg)
{
int ret = FI_SUCCESS, prev_state;
struct gnix_dgram_hndl *the_hndl = (struct gnix_dgram_hndl *)the_arg;
sigset_t sigmask;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
/*
* temporarily disable cancelability while we set up
* some stuff
*/
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &prev_state);
/*
* help out Cray core-spec, say we're not an app thread
* and can be run on core-spec cpus.
*/
ret = _gnix_task_is_not_app();
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_task_is_not_app call returned %d\n", ret);
/*
* block all signals, don't want this thread to catch
* signals that may be for app threads
*/
memset(&sigmask, 0, sizeof(sigset_t));
ret = sigfillset(&sigmask);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL,
"sigfillset call returned %d\n", ret);
} else {
ret = pthread_sigmask(SIG_SETMASK,
&sigmask, NULL);
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"pthread_sigmask call returned %d\n", ret);
}
/*
* okay now we're ready to be cancelable.
*/
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &prev_state);
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
retry:
ret = _gnix_dgram_poll(the_hndl, GNIX_DGRAM_BLOCK);
if ((ret == -FI_ETIMEDOUT) || (ret == FI_SUCCESS))
goto retry;
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_poll returned %s\n", fi_strerror(-ret));
/*
* TODO: need to be able to enqueue events on to the
* ep associated with the cm_nic.
*/
return NULL;
}
int _gnix_cm_nic_progress(void *arg)
{
struct gnix_cm_nic *cm_nic = (struct gnix_cm_nic *)arg;
int ret = FI_SUCCESS;
int complete;
struct gnix_work_req *p = NULL;
/*
* if we're doing FI_PROGRESS_MANUAL,
* see what's going on inside kgni's datagram
* box...
*/
if (cm_nic->ctrl_progress == FI_PROGRESS_MANUAL) {
++cm_nic->poll_cnt;
if (((cm_nic->poll_cnt % 512) == 0) ||
!dlist_empty(&cm_nic->cm_nic_wq)) {
ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
GNIX_DGRAM_NOBLOCK);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_poll returned %s\n",
fi_strerror(-ret));
goto err;
}
}
}
/*
* do a quick check if queue doesn't have anything yet,
* don't need this to be atomic
*/
check_again:
if (dlist_empty(&cm_nic->cm_nic_wq))
return ret;
/*
* okay, stuff to do, lock work queue,
* dequeue head, unlock, process work element,
* if it doesn't compete, put back at the tail
* of the queue.
*/
fastlock_acquire(&cm_nic->wq_lock);
p = dlist_first_entry(&cm_nic->cm_nic_wq, struct gnix_work_req,
list);
if (p == NULL) {
fastlock_release(&cm_nic->wq_lock);
return ret;
}
dlist_remove_init(&p->list);
fastlock_release(&cm_nic->wq_lock);
assert(p->progress_fn);
ret = p->progress_fn(p->data, &complete);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram prog fn returned %s\n",
fi_strerror(-ret));
}
if (complete == 1) {
if (p->completer_fn) {
ret = p->completer_fn(p->completer_data);
free(p);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram completer fn returned %s\n",
fi_strerror(-ret));
goto err;
}
} else {
free(p);
}
goto check_again;
} else {
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&p->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
}
err:
return ret;
}
