Test(utils, proc)
{
int rc;
rc = _gnix_task_is_not_app();
cr_expect(rc == 0);
/* *_unassigned_cpus flags don't work on tiger */
rc = _gnix_job_enable_unassigned_cpus();
cr_expect(rc != 0);
rc = _gnix_job_disable_unassigned_cpus();
cr_expect(rc != 0);
rc = _gnix_job_enable_affinity_apply();
cr_expect(rc == 0);
rc = _gnix_job_disable_affinity_apply();
cr_expect(rc == 0);
}
/*
* this function is intended to be invoked as an argument to pthread_create,
*/
static void *_gnix_dgram_prog_thread_fn(void *the_arg)
{
int ret = FI_SUCCESS, prev_state;
struct gnix_dgram_hndl *the_hndl = (struct gnix_dgram_hndl *)the_arg;
sigset_t sigmask;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
/*
* temporarily disable cancelability while we set up
* some stuff
*/
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &prev_state);
/*
* help out Cray core-spec, say we're not an app thread
* and can be run on core-spec cpus.
*/
ret = _gnix_task_is_not_app();
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_task_is_not_app call returned %d\n", ret);
/*
* block all signals, don't want this thread to catch
* signals that may be for app threads
*/
memset(&sigmask, 0, sizeof(sigset_t));
ret = sigfillset(&sigmask);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL,
"sigfillset call returned %d\n", ret);
} else {
ret = pthread_sigmask(SIG_SETMASK,
&sigmask, NULL);
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"pthread_sigmask call returned %d\n", ret);
}
/*
* okay now we're ready to be cancelable.
*/
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &prev_state);
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
retry:
ret = _gnix_dgram_poll(the_hndl, GNIX_DGRAM_BLOCK);
if ((ret == -FI_ETIMEDOUT) || (ret == FI_SUCCESS))
goto retry;
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_poll returned %s\n", fi_strerror(-ret));
/*
* TODO: need to be able to enqueue events on to the
* ep associated with the cm_nic.
*/
return NULL;
}
/*
* this function is intended to be invoked as an argument to pthread_create,
*/
static void *__gnix_nic_prog_thread_fn(void *the_arg)
{
int ret = FI_SUCCESS, prev_state;
int retry = 0;
uint32_t which;
struct gnix_nic *nic = (struct gnix_nic *)the_arg;
sigset_t sigmask;
gni_cq_handle_t cqv[2];
gni_return_t status;
gni_cq_entry_t cqe;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
/*
* temporarily disable cancelability while we set up
* some stuff
*/
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &prev_state);
/*
* help out Cray core-spec, say we're not an app thread
* and can be run on core-spec cpus.
*/
ret = _gnix_task_is_not_app();
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_task_is_not_app call returned %d\n",
ret);
/*
* block all signals, don't want this thread to catch
* signals that may be for app threads
*/
memset(&sigmask, 0, sizeof(sigset_t));
ret = sigfillset(&sigmask);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL,
"sigfillset call returned %d\n", ret);
} else {
ret = pthread_sigmask(SIG_SETMASK,
&sigmask, NULL);
if (ret)
GNIX_WARN(FI_LOG_EP_CTRL,
"pthread_sigmask call returned %d\n", ret);
}
/*
* okay now we're ready to be cancelable.
*/
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &prev_state);
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
cqv[0] = nic->tx_cq_blk;
cqv[1] = nic->rx_cq_blk;
try_again:
status = GNI_CqVectorMonitor(cqv,
2,
-1,
&which);
switch (status) {
case GNI_RC_SUCCESS:
/*
* first dequeue RX CQEs
*/
if (which == 1) {
do {
status = GNI_CqGetEvent(nic->rx_cq_blk,
&cqe);
} while (status == GNI_RC_SUCCESS);
}
_gnix_nic_progress(nic);
retry = 1;
break;
case GNI_RC_TIMEOUT:
retry = 1;
break;
case GNI_RC_NOT_DONE:
retry = 1;
break;
case GNI_RC_INVALID_PARAM:
case GNI_RC_INVALID_STATE:
case GNI_RC_ERROR_RESOURCE:
case GNI_RC_ERROR_NOMEM:
retry = 0;
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqGetEvent returned %s\n",
gni_err_str[status]);
break;
default:
retry = 0;
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqGetEvent returned unexpected code %s\n",
gni_err_str[status]);
break;
}
if (retry)
goto try_again;
return NULL;
}
