static inline lagopus_result_t
s_start_callout_main_loop(void) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
global_state_t s;
shutdown_grace_level_t l;
ret = global_state_wait_for(GLOBAL_STATE_STARTED, &s, &l, -1LL);
if (likely(ret == LAGOPUS_RESULT_OK)) {
if (likely(s == GLOBAL_STATE_STARTED)) {
#ifdef CO_MSG_DEBUG
lagopus_chrono_t timeout = s_idle_interval;
#else
lagopus_chrono_t timeout;
#endif /* CO_MSG_DEBUG */
lagopus_callout_task_t out_tasks[CALLOUT_TASK_MAX * 3];
size_t n_out_tasks;
lagopus_callout_task_t urgent_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_urgent_tasks;
lagopus_callout_task_t idle_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_idle_tasks;
lagopus_callout_task_t timed_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_timed_tasks;
lagopus_result_t r;
lagopus_chrono_t now;
lagopus_chrono_t next_wakeup;
lagopus_chrono_t prev_wakeup;
int cstate = 0;
WHAT_TIME_IS_IT_NOW_IN_NSEC(prev_wakeup);
(void)lagopus_mutex_enter_critical(&s_sched_lck, &cstate);
{
s_is_stopped = false;
mbar();
while (s_do_loop == true) {
n_out_tasks = 0;
/*
* Get the current time.
*/
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(3, "now: " PF64(d) "\n", now);
lagopus_msg_debug(3, "prv: " PF64(d) "\n", prev_wakeup);
lagopus_msg_debug(3, "to: " PF64(d) "\n", timeout);
#endif /* CO_MSG_DEBUG */
s_lock_global();
{
/*
* Acquire the global lock to make the task
* submisson/fetch atomic.
*/
sn_urgent_tasks =
lagopus_bbq_get_n(&s_urgent_tsk_q, (void **)urgent_tasks,
CALLOUT_TASK_MAX, 1LL,
lagopus_callout_task_t,
0LL, NULL);
sn_idle_tasks =
lagopus_bbq_get_n(&s_idle_tsk_q, (void **)idle_tasks,
CALLOUT_TASK_MAX, 1LL,
lagopus_callout_task_t,
0LL, NULL);
}
s_unlock_global();
/*
* Pack the tasks into a buffer.
*/
sn_timed_tasks = s_get_runnable_timed_task(now, timed_tasks,
CALLOUT_TASK_MAX,
&next_wakeup);
if (sn_timed_tasks > 0) {
/*
* Pack the timed tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
timed_tasks,
(size_t)(sn_timed_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_timed_tasks;
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(3, "timed task " PF64(u) ".\n",
sn_timed_tasks);
lagopus_msg_debug(3, "nw: " PF64(d) ".\n",
next_wakeup);
#endif /* CO_MSG_DEBUG */
} else if (sn_timed_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_timed_tasks);
lagopus_msg_error("timed tasks fetch failed.\n");
}
if (sn_urgent_tasks > 0) {
/*
* Pack the urgent tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
urgent_tasks,
(size_t)(sn_urgent_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_urgent_tasks;
} else if (sn_urgent_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_urgent_tasks);
lagopus_msg_error("urgent tasks fetch failed.\n");
}
if (sn_idle_tasks > 0) {
/*
* Pack the idle tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
idle_tasks,
(size_t)(sn_idle_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_idle_tasks;
} else if (sn_idle_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_idle_tasks);
lagopus_msg_error("idle tasks fetch failed.\n");
}
if (n_out_tasks > 0) {
/*
* Run/Submit the tasks.
*/
r = (s_final_task_sched_proc)(out_tasks, now, n_out_tasks);
if (unlikely(r <= 0)) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(r);
lagopus_msg_error("failed to submit " PFSZ(u)
" urgent/timed tasks.\n", n_out_tasks);
}
}
if (s_idle_proc != NULL &&
s_next_idle_abstime < (now + CALLOUT_TASK_SCHED_JITTER)) {
if (likely(s_idle_proc(s_idle_proc_arg) ==
LAGOPUS_RESULT_OK)) {
s_next_idle_abstime = now + s_idle_interval;
} else {
/*
* Stop the main loop and return (clean finish.)
*/
s_do_loop = false;
goto critical_end;
}
}
/*
* fetch the start time of the timed task in the queue head.
*/
next_wakeup = s_peek_current_wakeup_time();
if (next_wakeup <= 0LL) {
/*
* Nothing in the timed Q.
*/
if (s_next_idle_abstime <= 0LL) {
s_next_idle_abstime = now + s_idle_interval;
}
next_wakeup = s_next_idle_abstime;
}
/*
* TODO
*
* Re-optimize forcible waje up by timed task submission
* timing and times. See also
* callout_queue.c:s_do_sched().
*/
/*
* calculate the timeout and sleep.
*/
timeout = next_wakeup - now;
if (likely(timeout > 0LL)) {
if (timeout > s_idle_interval) {
timeout = s_idle_interval;
next_wakeup = now + timeout;
}
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4,
"about to sleep, timeout " PF64(d) " nsec.\n",
timeout);
#endif /* CO_MSG_DEBUG */
prev_wakeup = next_wakeup;
r = lagopus_bbq_wait_gettable(&s_urgent_tsk_q, timeout);
if (unlikely(r <= 0 &&
r != LAGOPUS_RESULT_TIMEDOUT &&
r != LAGOPUS_RESULT_WAKEUP_REQUESTED)) {
lagopus_perror(r);
lagopus_msg_error("Event wait failure.\n");
ret = r;
goto critical_end;
} else {
if (r == LAGOPUS_RESULT_WAKEUP_REQUESTED) {
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4, "woke up.\n");
#endif /* CO_MSG_DEBUG */
}
}
} else {
WHAT_TIME_IS_IT_NOW_IN_NSEC(next_wakeup);
prev_wakeup = next_wakeup;
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4, "timeout zero. contiune.\n");
#endif /* CO_MSG_DEBUG */
}
/*
* The end of the desired potion of the loop.
*/
} /* while (s_do_loop == true) */
}
critical_end:
s_is_stopped = true;
s_wakeup_sched();
(void)lagopus_mutex_leave_critical(&s_sched_lck, cstate);
if (s_do_loop == false) {
/*
* The clean finish.
*/
ret = LAGOPUS_RESULT_OK;
}
} else { /* s == GLOBAL_STATE_STARTED */
s_is_stopped = true;
ret = LAGOPUS_RESULT_INVALID_STATE_TRANSITION;
}
} else {
s_is_stopped = true;
}
return ret;
}
static int
do_run(size_t nthds, ssize_t nputs) {
int ret = 1;
lagopus_result_t r;
size_t i;
size_t j;
char thdname[16];
lagopus_chrono_t *put_dates = NULL;
size_t n_need_watch = 0;
size_t n_valid_polls = 0;
ssize_t qsz;
test_thread_t tt;
lagopus_bbq_t bbq;
lagopus_chrono_t t_begin;
lagopus_chrono_t t_end;
lagopus_chrono_t p_begin;
lagopus_chrono_t p_t;
ssize_t n_gets = 0;
lagopus_chrono_t p_min = LLONG_MAX;
lagopus_chrono_t p_max = LLONG_MIN;
double p_sum = 0.0;
double p_sum2 = 0.0;
double p_avg;
double p_sd;
lagopus_chrono_t t_total = 0;
double t_avg;
/*
* This is only for choking clang/scan-build.
*/
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
put_dates = (lagopus_chrono_t *)
malloc(sizeof(lagopus_chrono_t) * (size_t)nputs);
if (put_dates == NULL) {
goto done;
}
if ((r = lagopus_mutex_create(&start_lock)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
if ((r = lagopus_mutex_create(&stop_lock)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
/*
* Create the qmuxer.
*/
if ((r = lagopus_qmuxer_create(&qmx)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
/*
* Then create queues.
*/
bbqs = (lagopus_bbq_t *)malloc(sizeof(lagopus_bbq_t) * nthds);
if (bbqs == NULL) {
goto done;
}
for (i = 0; i < nthds; i++) {
if ((r = lagopus_bbq_create(&(bbqs[i]), lagopus_chrono_t,
1000LL * 1000LL,
NULL)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
n_created_bbqs++;
}
if (n_created_bbqs == 0) {
goto done;
}
/*
* Then create poll objects for the each queue.
*/
polls = (lagopus_qmuxer_poll_t *)malloc(sizeof(lagopus_qmuxer_poll_t) *
n_created_bbqs);
if (polls == NULL) {
goto done;
}
for (i = 0; i < n_created_bbqs; i++) {
if ((r = lagopus_qmuxer_poll_create(&(polls[i]),
bbqs[i],
LAGOPUS_QMUXER_POLL_READABLE)) !=
LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
n_created_polls++;
}
if (n_created_polls == 0) {
goto done;
}
/*
* Then create threads for the each poll objects/queues.
*/
tts = (test_thread_record *)malloc(sizeof(test_thread_record) *
n_created_polls);
if (tts == NULL) {
goto done;
}
for (i = 0; i < n_created_polls; i++) {
snprintf(thdname, sizeof(thdname), "putter " PFSZS(4, u), i);
tt = &(tts[i]);
if (test_thread_create(&tt, start_lock, bbqs[i], nputs,
(const char *)thdname) != true) {
goto done;
}
n_created_thds++;
}
if (n_created_thds == 0) {
goto done;
}
/*
* Let the initiation begin.
*/
/*
* Ready, note that all the created threads do this lock.
*/
(void)lagopus_mutex_lock(&start_lock);
/*
* Steady,
*/
for (i = 0; i < n_created_thds; i++) {
tt = &(tts[i]);
if (lagopus_thread_start((lagopus_thread_t *)&tt, false) !=
LAGOPUS_RESULT_OK) {
(void)lagopus_mutex_unlock(&start_lock);
goto done;
}
}
fprintf(stdout, "Test for " PFSZ(u) " threads " PFSZ(u)
" events/thdread start.\n", n_created_thds, (size_t)nputs);
/*
* Go.
*/
(void)lagopus_mutex_unlock(&start_lock);
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
while (true) {
/*
* Like the select(2)/poll(2), initialize poll objects before
* checking events.
*/
n_need_watch = 0;
n_valid_polls = 0;
for (i = 0; i < n_created_thds; i++) {
/*
* Check if the poll has a valid queue.
*/
bbq = NULL;
if ((r = lagopus_qmuxer_poll_get_queue(&(polls[i]), &bbq)) !=
LAGOPUS_RESULT_OK) {
lagopus_perror(r);
break;
}
if (bbq != NULL) {
n_valid_polls++;
}
/*
* Reset the poll status.
*/
if ((r = lagopus_qmuxer_poll_reset(&(polls[i]))) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
break;
}
n_need_watch++;
}
/*
* If there are no valid queues, exit.
*/
if (n_valid_polls == 0) {
break;
}
/*
* Wait for an event.
*
* Note that we better set timeout, not waiting forever.
*/
r = lagopus_qmuxer_poll(&qmx, (lagopus_qmuxer_poll_t *const)polls,
n_need_watch,
100LL * 1000LL * 1000LL);
if (r > 0) {
/*
* Check which poll got an event. Actually, check all the queues
* in this sample.
*/
size_t n_actual_get = 0LL;
for (i = 0; i < n_created_thds; i++) {
if ((qsz = lagopus_bbq_size(&(bbqs[i]))) > 0) {
lagopus_msg_debug(1, "Got " PFSZS(8, u) " events from the Q"
PFSZS(03, u) ".\n",
(size_t)qsz, (size_t)i);
if ((r = lagopus_bbq_get_n(&(bbqs[i]), (void **)put_dates,
(size_t)nputs, 1LL,
lagopus_chrono_t,
1000LL * 1000LL * 1000LL,
&n_actual_get)) > 0) {
#if 1
WHAT_TIME_IS_IT_NOW_IN_NSEC(p_begin);
#endif
for (j = 0; j < n_actual_get; j++) {
/*
* In this sample, -1LL is kinda 'EOF'. Check if we got
* the EOF.
*/
if (put_dates[j] == -1LL) {
/*
* The queue is kinda 'closed'. From now on we don't
* check this queue anymore. To specify this:
*/
lagopus_msg_debug(1, "Got an EOF from the Q" PFSZS(04, u)
".\n", i);
goto nullify;
}
#if 0
WHAT_TIME_IS_IT_NOW_IN_NSEC(p_begin);
#endif
p_t = p_begin - put_dates[j];
if (p_t < p_min) {
p_min = p_t;
}
if (p_t > p_max) {
p_max = p_t;
}
p_sum += (double)p_t;
p_sum2 += ((double)p_t * (double)p_t);
n_gets++;
}
} else {
/*
* Something wrong for the queue. But we must not exit
* here. Keep on checking other queues. In order to do
* this, set NULL as the queue into the poll object for
* the queue.
*/
lagopus_perror(r);
nullify:
if ((r = lagopus_qmuxer_poll_set_queue(&(polls[i]), NULL)) ==
LAGOPUS_RESULT_OK) {
lagopus_msg_debug(1, "Q" PFSZS(04, u) " is not valid "
"anymore, ignore the queue.\n", i);
break;
} else {
/*
* There is nothing we can do now.
*/
lagopus_perror(r);
goto done;
}
}
}
}
} else if (r == LAGOPUS_RESULT_TIMEDOUT) {
lagopus_msg_debug(1, "Timedout. continue.\n");
continue;
} else {
lagopus_perror(r);
lagopus_msg_debug(1, "Break the loop due to error(s).\n");
goto done;
}
}
ret = 0;
done:
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_end);
if (is_signaled == false) {
fprintf(stdout, "Done.\n");
} else {
fprintf(stdout, "Stopped.\n");
}
fprintf(stdout, "Total # of the events:\t" PFSZS(22, u) "\n\n", n_gets);
p_avg = p_sum / (double)n_gets;
p_sd = (p_sum2 -
2.0 * p_avg * p_sum +
p_avg * p_avg * (double)n_gets) / (double)(n_gets - 1);
p_sd = sqrt(p_sd);
fprintf(stdout, "Queue stats:\n");
fprintf(stdout, "wait time min =\t" PFSZS(22, d) " nsec.\n", p_min);
fprintf(stdout, "wait time max =\t" PFSZS(22, d) " nsec.\n", p_max);
fprintf(stdout, "wait time avg =\t%25.2f nsec.\n", p_avg);
fprintf(stdout, "wait time sd =\t%25.2f.\n\n", p_sd);
t_total = t_end - t_begin;
t_avg = (double)t_total / (double)n_gets;
fprintf(stdout, "Throughput:\n");
fprintf(stdout, "total time:\t" PFSZS(22, d) " msec.\n",
(size_t)(t_total / 1000LL / 1000LL));
fprintf(stdout, "total avg:\t%25.2f nsec/event.\n", t_avg);
s_destroy_all();
free((void *)put_dates);
return ret;
}
