static lagopus_result_t
s_test_thread_main(const lagopus_thread_t *tptr, void *arg) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
lagopus_chrono_t *dates = NULL;
(void)arg;
if (tptr != NULL) {
test_thread_t tt = (test_thread_t)*tptr;
if (tt != NULL) {
ssize_t i;
lagopus_chrono_t now;
dates = (lagopus_chrono_t *)
malloc(sizeof(lagopus_chrono_t) * (size_t)tt->m_n_puts);
if (dates == NULL) {
goto done;
}
/*
* Ready, steady,
*/
(void)lagopus_mutex_lock(&(tt->m_start_lock));
/*
* go.
*/
(void)lagopus_mutex_unlock(&(tt->m_start_lock));
for (i = 0; i < tt->m_n_puts; i++) {
WHAT_TIME_IS_IT_NOW_IN_NSEC(dates[i]);
}
if (lagopus_bbq_put_n(&(tt->m_q), (void **)dates,
(size_t)tt->m_n_puts,
lagopus_chrono_t, -1LL, NULL) !=
(lagopus_result_t)tt->m_n_puts) {
goto done;
}
now = -1LL;
if (lagopus_bbq_put(&(tt->m_q), &now, lagopus_chrono_t, -1LL) !=
LAGOPUS_RESULT_OK) {
goto done;
}
ret = tt->m_n_puts;
lagopus_msg_debug(1, "Done, ret = " PFSZS(020, d)
", req = " PFSZS(020, u) ".\n",
(size_t)ret, (size_t)tt->m_n_puts);
}
}
done:
free((void *)dates);
return ret;
}
int
main(int argc, const char *const argv[]) {
size_t i;
uint64_t zz = 0;
struct timespec ts0;
struct timespec ts1;
struct timespec ts2;
lagopus_chrono_t now;
uint64_t start0, start1, start2, end0, end1, end2;
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
NSEC_TO_TS(now, ts0);
NSEC_TO_TS_MOD(now, ts1);
NSEC_TO_TS_ASM(now, ts2);
fprintf(stderr, "ts0: " PF64(u) ", " PF64(u) "\n",
ts0.tv_sec, ts0.tv_nsec);
fprintf(stderr, "ts1: " PF64(u) ", " PF64(u) "\n",
ts1.tv_sec, ts1.tv_nsec);
fprintf(stderr, "ts2: " PF64(u) ", " PF64(u) "\n",
ts2.tv_sec, ts2.tv_nsec);
for (i = 0; i < 1000LL * 1000LL * 1000LL; i++) {
zz += i;
}
fprintf(stderr, "zz = " PF64(u) "\n", zz);
start0 = lagopus_rdtsc();
for (i = 0; i < 1000LL * 1000LL * 1000LL; i++) {
NSEC_TO_TS(i, ts0);
}
end0 = lagopus_rdtsc();
start1 = lagopus_rdtsc();
for (i = 0; i < 1000LL * 1000LL * 1000LL; i++) {
NSEC_TO_TS_MOD(i, ts1);
}
end1 = lagopus_rdtsc();
start2 = lagopus_rdtsc();
for (i = 0; i < 1000LL * 1000LL * 1000LL; i++) {
NSEC_TO_TS_ASM(i, ts0);
}
end2 = lagopus_rdtsc();
fprintf(stderr, "cur: " PF64(u) "\n", end0 - start0);
fprintf(stderr, "mod: " PF64(u) "\n", end1 - start1);
fprintf(stderr, "asm: " PF64(u) "\n", end2 - start2);
return 0;
}
static gallus_result_t
s_ingress_main(const gallus_pipeline_stage_t *sptr,
size_t idx,
void *evbuf,
size_t n_evs) {
gallus_result_t ret = GALLUS_RESULT_ANY_FAILURES;
(void)evbuf;
(void)n_evs;
if (likely(sptr != NULL && *sptr != NULL)) {
test_stage_t ts = (test_stage_t)(*sptr);
base_stage_t bs = (base_stage_t)ts;
if (likely(bs != NULL && bs->m_next_stg != NULL)) {
if (likely(ts->m_states[idx] != test_stage_state_done)) {
gallus_chrono_t start_time;
uint64_t start_clock;
uint64_t end_clock;
uint64_t *addr = ts->m_data + ts->m_enq_infos[idx].m_offset;
size_t len = ts->m_enq_infos[idx].m_length;
WHAT_TIME_IS_IT_NOW_IN_NSEC(start_time);
start_clock = gallus_rdtsc();
ret = gallus_pipeline_stage_submit((gallus_pipeline_stage_t *)
&(bs->m_next_stg),
(void *)addr, len, (void *)idx);
end_clock = gallus_rdtsc();
gallus_atomic_update_min(uint64_t, &(ts->m_start_clock),
0, start_clock);
gallus_atomic_update_max(uint64_t, &(ts->m_end_clock),
0, end_clock);
gallus_atomic_update_min(gallus_chrono_t, &(ts->m_start_time),
-1LL, start_time);
ts->m_states[idx] = test_stage_state_done;
mbar();
} else {
ret = s_test_stage_wait(ts);
}
} else {
ret = GALLUS_RESULT_INVALID_ARGS;
}
} else {
ret = GALLUS_RESULT_INVALID_ARGS;
}
return ret;
}
lagopus_result_t
lagopus_cond_wait(lagopus_cond_t *cndptr,
lagopus_mutex_t *mtxptr,
lagopus_chrono_t nsec) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
if (mtxptr != NULL &&
*mtxptr != NULL &&
cndptr != NULL &&
*cndptr != NULL) {
int st;
errno = 0;
if (nsec < 0) {
if ((st = pthread_cond_wait(&((*cndptr)->m_cond),
&((*mtxptr)->m_mtx))) == 0) {
ret = LAGOPUS_RESULT_OK;
} else {
errno = st;
ret = LAGOPUS_RESULT_POSIX_API_ERROR;
}
} else {
struct timespec ts;
lagopus_chrono_t now;
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
now += nsec;
NSEC_TO_TS(now, ts);
retry:
errno = 0;
if ((st = pthread_cond_timedwait(&((*cndptr)->m_cond),
&((*mtxptr)->m_mtx),
&ts)) == 0) {
ret = LAGOPUS_RESULT_OK;
} else {
if (st == EINTR) {
goto retry;
} else if (st == ETIMEDOUT) {
ret = LAGOPUS_RESULT_TIMEDOUT;
} else {
errno = st;
ret = LAGOPUS_RESULT_POSIX_API_ERROR;
}
}
}
} else {
ret = LAGOPUS_RESULT_INVALID_ARGS;
}
return ret;
}
static inline lagopus_result_t
s_wait_stage(lagopus_pipeline_stage_t ps, size_t n, lagopus_chrono_t nsec,
bool is_in_destroy) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
if (ps != NULL) {
size_t i;
if (is_in_destroy == false && nsec > 0) {
lagopus_chrono_t w_begin;
lagopus_chrono_t w_end;
lagopus_chrono_t w = nsec;
for (ret = LAGOPUS_RESULT_OK, i = 0;
i < n && ret == LAGOPUS_RESULT_OK && w > 0LL;
i++) {
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_begin);
ret = s_worker_wait(&(ps->m_workers[i]), w);
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_end);
if (ret == LAGOPUS_RESULT_OK) {
w = w_end - w_begin;
}
}
} else {
for (i = 0; i < n; i++) {
(void)s_worker_wait(&(ps->m_workers[i]), -1LL);
}
ret = LAGOPUS_RESULT_OK;
}
} else {
ret = LAGOPUS_RESULT_INVALID_ARGS;
}
return ret;
}
lagopus_result_t
lagopus_rwlock_writer_timedlock(lagopus_rwlock_t *rwlptr,
lagopus_chrono_t nsec) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
if (rwlptr != NULL &&
*rwlptr != NULL) {
int st;
if (nsec < 0) {
if ((st = pthread_rwlock_wrlock(&((*rwlptr)->m_rwl))) == 0) {
(*rwlptr)->m_prev_cancel_state = -INT_MAX;
ret = LAGOPUS_RESULT_OK;
} else {
errno = st;
ret = LAGOPUS_RESULT_POSIX_API_ERROR;
}
} else {
struct timespec ts;
lagopus_chrono_t now;
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
now += nsec;
NSEC_TO_TS(now, ts);
if ((st = pthread_rwlock_timedwrlock(&((*rwlptr)->m_rwl),
&ts)) == 0) {
(*rwlptr)->m_prev_cancel_state = -INT_MAX;
ret = LAGOPUS_RESULT_OK;
} else {
errno = st;
if (st == ETIMEDOUT) {
ret = LAGOPUS_RESULT_TIMEDOUT;
} else {
ret = LAGOPUS_RESULT_POSIX_API_ERROR;
}
}
}
} else {
ret = LAGOPUS_RESULT_INVALID_ARGS;
}
return ret;
}
static lagopus_result_t
callout_task(void *arg) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
lagopus_chrono_t now;
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
lagopus_msg_debug(1, "enter.\n");
if (likely(arg != NULL)) {
callout_arg_t carg = (callout_arg_t)arg;
size_t n_exec = __sync_add_and_fetch(&(carg->m_n_exec), 1);
lagopus_msg_debug(1, "exec " PFSZ(u) ".\n", n_exec);
if (carg->m_last_exec_abstime != 0) {
lagopus_msg_debug(1, "interval: " PF64(d) " nsec.\n",
now - carg->m_last_exec_abstime);
}
carg->m_last_exec_abstime = now;
if (carg->m_wait_nsec > 0) {
lagopus_msg_debug(1, "sleep " PFSZ(u) " nsec.\n",
carg->m_wait_nsec);
(void)lagopus_chrono_nanosleep(carg->m_wait_nsec, NULL);
}
if (n_exec < carg->m_n_stop) {
ret = LAGOPUS_RESULT_OK;
}
}
lagopus_msg_debug(1, "leave.\n");
return ret;
}
gallus_result_t
gallus_module_wait_all(gallus_chrono_t nsec) {
gallus_result_t ret = GALLUS_RESULT_ANY_FAILURES;
s_lock();
{
s_gstate = MODULE_GLOBAL_STATE_WAITING;
if (s_n_modules > 0) {
gallus_result_t first_err = GALLUS_RESULT_OK;
size_t i;
a_module *mptr;
/*
* Reverse order.
*/
if (nsec < 0LL) {
for (i = 0; i < s_n_modules; i++) {
mptr = &(s_modules[s_n_modules - i - 1]);
ret = s_wait_module(mptr, -1LL);
if (ret != GALLUS_RESULT_OK) {
gallus_perror(ret);
gallus_msg_error("can't wait module \"%s\".\n",
mptr->m_name);
if (first_err == GALLUS_RESULT_OK) {
first_err = ret;
}
}
/*
* Just carry on wait no matter what kind of errors
* occur.
*/
}
} else {
gallus_chrono_t w_begin;
gallus_chrono_t w_end;
gallus_chrono_t w = nsec;
for (i = 0; i < s_n_modules; i++) {
mptr = &(s_modules[s_n_modules - i - 1]);
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_begin);
ret = s_wait_module(mptr, w);
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_end);
if (ret != GALLUS_RESULT_OK) {
gallus_perror(ret);
gallus_msg_error("can't wait module \"%s\".\n",
mptr->m_name);
if (first_err == GALLUS_RESULT_OK) {
first_err = ret;
}
}
/*
* Just carry on wait no matter what kind of errors
* occur.
*/
w = nsec - (w_end - w_begin);
if (w < 0LL) {
w = 0LL;
}
}
}
ret = first_err;
} else {
ret = GALLUS_RESULT_OK;
}
if (ret == GALLUS_RESULT_OK) {
s_gstate = MODULE_GLOBAL_STATE_SHUTDOWN;
}
}
s_unlock();
return ret;
}
static inline lagopus_result_t
s_start_callout_main_loop(void) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
global_state_t s;
shutdown_grace_level_t l;
ret = global_state_wait_for(GLOBAL_STATE_STARTED, &s, &l, -1LL);
if (likely(ret == LAGOPUS_RESULT_OK)) {
if (likely(s == GLOBAL_STATE_STARTED)) {
#ifdef CO_MSG_DEBUG
lagopus_chrono_t timeout = s_idle_interval;
#else
lagopus_chrono_t timeout;
#endif /* CO_MSG_DEBUG */
lagopus_callout_task_t out_tasks[CALLOUT_TASK_MAX * 3];
size_t n_out_tasks;
lagopus_callout_task_t urgent_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_urgent_tasks;
lagopus_callout_task_t idle_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_idle_tasks;
lagopus_callout_task_t timed_tasks[CALLOUT_TASK_MAX];
lagopus_result_t sn_timed_tasks;
lagopus_result_t r;
lagopus_chrono_t now;
lagopus_chrono_t next_wakeup;
lagopus_chrono_t prev_wakeup;
int cstate = 0;
WHAT_TIME_IS_IT_NOW_IN_NSEC(prev_wakeup);
(void)lagopus_mutex_enter_critical(&s_sched_lck, &cstate);
{
s_is_stopped = false;
mbar();
while (s_do_loop == true) {
n_out_tasks = 0;
/*
* Get the current time.
*/
WHAT_TIME_IS_IT_NOW_IN_NSEC(now);
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(3, "now: " PF64(d) "\n", now);
lagopus_msg_debug(3, "prv: " PF64(d) "\n", prev_wakeup);
lagopus_msg_debug(3, "to: " PF64(d) "\n", timeout);
#endif /* CO_MSG_DEBUG */
s_lock_global();
{
/*
* Acquire the global lock to make the task
* submisson/fetch atomic.
*/
sn_urgent_tasks =
lagopus_bbq_get_n(&s_urgent_tsk_q, (void **)urgent_tasks,
CALLOUT_TASK_MAX, 1LL,
lagopus_callout_task_t,
0LL, NULL);
sn_idle_tasks =
lagopus_bbq_get_n(&s_idle_tsk_q, (void **)idle_tasks,
CALLOUT_TASK_MAX, 1LL,
lagopus_callout_task_t,
0LL, NULL);
}
s_unlock_global();
/*
* Pack the tasks into a buffer.
*/
sn_timed_tasks = s_get_runnable_timed_task(now, timed_tasks,
CALLOUT_TASK_MAX,
&next_wakeup);
if (sn_timed_tasks > 0) {
/*
* Pack the timed tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
timed_tasks,
(size_t)(sn_timed_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_timed_tasks;
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(3, "timed task " PF64(u) ".\n",
sn_timed_tasks);
lagopus_msg_debug(3, "nw: " PF64(d) ".\n",
next_wakeup);
#endif /* CO_MSG_DEBUG */
} else if (sn_timed_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_timed_tasks);
lagopus_msg_error("timed tasks fetch failed.\n");
}
if (sn_urgent_tasks > 0) {
/*
* Pack the urgent tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
urgent_tasks,
(size_t)(sn_urgent_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_urgent_tasks;
} else if (sn_urgent_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_urgent_tasks);
lagopus_msg_error("urgent tasks fetch failed.\n");
}
if (sn_idle_tasks > 0) {
/*
* Pack the idle tasks.
*/
(void)memcpy((void *)(out_tasks + n_out_tasks),
idle_tasks,
(size_t)(sn_idle_tasks) *
sizeof(lagopus_callout_task_t));
n_out_tasks += (size_t)sn_idle_tasks;
} else if (sn_idle_tasks < 0) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(sn_idle_tasks);
lagopus_msg_error("idle tasks fetch failed.\n");
}
if (n_out_tasks > 0) {
/*
* Run/Submit the tasks.
*/
r = (s_final_task_sched_proc)(out_tasks, now, n_out_tasks);
if (unlikely(r <= 0)) {
/*
* We can't be treat this as a fatal error. Carry on.
*/
lagopus_perror(r);
lagopus_msg_error("failed to submit " PFSZ(u)
" urgent/timed tasks.\n", n_out_tasks);
}
}
if (s_idle_proc != NULL &&
s_next_idle_abstime < (now + CALLOUT_TASK_SCHED_JITTER)) {
if (likely(s_idle_proc(s_idle_proc_arg) ==
LAGOPUS_RESULT_OK)) {
s_next_idle_abstime = now + s_idle_interval;
} else {
/*
* Stop the main loop and return (clean finish.)
*/
s_do_loop = false;
goto critical_end;
}
}
/*
* fetch the start time of the timed task in the queue head.
*/
next_wakeup = s_peek_current_wakeup_time();
if (next_wakeup <= 0LL) {
/*
* Nothing in the timed Q.
*/
if (s_next_idle_abstime <= 0LL) {
s_next_idle_abstime = now + s_idle_interval;
}
next_wakeup = s_next_idle_abstime;
}
/*
* TODO
*
* Re-optimize forcible waje up by timed task submission
* timing and times. See also
* callout_queue.c:s_do_sched().
*/
/*
* calculate the timeout and sleep.
*/
timeout = next_wakeup - now;
if (likely(timeout > 0LL)) {
if (timeout > s_idle_interval) {
timeout = s_idle_interval;
next_wakeup = now + timeout;
}
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4,
"about to sleep, timeout " PF64(d) " nsec.\n",
timeout);
#endif /* CO_MSG_DEBUG */
prev_wakeup = next_wakeup;
r = lagopus_bbq_wait_gettable(&s_urgent_tsk_q, timeout);
if (unlikely(r <= 0 &&
r != LAGOPUS_RESULT_TIMEDOUT &&
r != LAGOPUS_RESULT_WAKEUP_REQUESTED)) {
lagopus_perror(r);
lagopus_msg_error("Event wait failure.\n");
ret = r;
goto critical_end;
} else {
if (r == LAGOPUS_RESULT_WAKEUP_REQUESTED) {
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4, "woke up.\n");
#endif /* CO_MSG_DEBUG */
}
}
} else {
WHAT_TIME_IS_IT_NOW_IN_NSEC(next_wakeup);
prev_wakeup = next_wakeup;
#ifdef CO_MSG_DEBUG
lagopus_msg_debug(4, "timeout zero. contiune.\n");
#endif /* CO_MSG_DEBUG */
}
/*
* The end of the desired potion of the loop.
*/
} /* while (s_do_loop == true) */
}
critical_end:
s_is_stopped = true;
s_wakeup_sched();
(void)lagopus_mutex_leave_critical(&s_sched_lck, cstate);
if (s_do_loop == false) {
/*
* The clean finish.
*/
ret = LAGOPUS_RESULT_OK;
}
} else { /* s == GLOBAL_STATE_STARTED */
s_is_stopped = true;
ret = LAGOPUS_RESULT_INVALID_STATE_TRANSITION;
}
} else {
s_is_stopped = true;
}
return ret;
}
static inline lagopus_chrono_t
s_do_sched(lagopus_callout_task_t t) {
lagopus_result_t ret = -1LL;
lagopus_result_t r;
lagopus_callout_task_t e;
s_lock_task_q();
{
s_lock_task(t);
{
if (likely(t->m_is_in_timed_q == false)) {
/*
* Firstly (re-)compute the next exeution time of this
* task.
*/
if (t->m_is_first == false && t->m_do_repeat == true) {
t->m_next_abstime = t->m_last_abstime + t->m_interval_time;
} else {
WHAT_TIME_IS_IT_NOW_IN_NSEC(t->m_last_abstime);
t->m_next_abstime = t->m_last_abstime + t->m_initial_delay_time;
}
/*
* Then insert the task into the Q.
*/
e = TAILQ_FIRST(&s_chrono_tsk_q);
if (e != NULL) {
if (e->m_next_abstime > t->m_next_abstime) {
TAILQ_INSERT_HEAD(&s_chrono_tsk_q, t, m_entry);
} else {
for (;
e != NULL && e->m_next_abstime <= t->m_next_abstime;
e = TAILQ_NEXT(e, m_entry)) {
;
}
if (e != NULL) {
TAILQ_INSERT_BEFORE(e, t, m_entry);
} else {
TAILQ_INSERT_TAIL(&s_chrono_tsk_q, t, m_entry);
}
}
} else {
TAILQ_INSERT_TAIL(&s_chrono_tsk_q, t, m_entry);
}
(void)s_set_task_state_in_table(t, TASK_STATE_ENQUEUED);
t->m_status = TASK_STATE_ENQUEUED;
t->m_is_in_timed_q = true;
ret = t->m_next_abstime;
/*
* Fianlly wake the master scheduler.
*/
lagopus_msg_debug(4, "wake the master scheduler up.\n");
/*
* TODO
*
* Re-optimize forcible wake up by timed task submission
* timing and times. See also
* callout.c:s_start_callout_main_loop()
*/
r = lagopus_bbq_wakeup(&s_urgent_tsk_q, 0LL);
if (unlikely(r != LAGOPUS_RESULT_OK)) {
lagopus_perror(r);
lagopus_msg_error("can't wake the callout task master "
"scheduler up.\n");
}
}
}
s_unlock_task(t);
}
s_unlock_task_q();
return ret;
}
static inline lagopus_result_t
s_get_n(lagopus_cbuffer_t *cbptr,
void *valptr,
size_t n_vals_max,
size_t n_at_least,
size_t valsz,
lagopus_chrono_t nsec,
size_t *n_actual_get,
bool do_incr) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
lagopus_cbuffer_t cb = NULL;
if (cbptr != NULL && (cb = *cbptr) != NULL &&
valptr != NULL &&
valsz == cb->m_element_size) {
if (n_vals_max > 0) {
int64_t n_copyout = 0LL;
if (nsec == 0LL) {
s_lock(cb);
{
/*
* Just get and return.
*/
if (cb->m_is_operational == true) {
n_copyout = ret = s_copyout(cb, valptr, n_vals_max, do_incr);
} else {
ret = LAGOPUS_RESULT_NOT_OPERATIONAL;
}
}
s_unlock(cb);
} else if (nsec < 0LL) {
s_lock(cb);
{
/*
* Repeat getting until all the required # of the data are got.
*/
check_inf:
mbar();
if (cb->m_is_operational == true) {
n_copyout += s_copyout(cb,
(void *)((char *)valptr +
((size_t)n_copyout * valsz)),
n_vals_max - (size_t)n_copyout,
do_incr);
if ((size_t)n_copyout < n_vals_max) {
/*
* Need to repeat.
*/
if (cb->m_n_elements < 1LL) {
/*
* No data. Need to wait for someone put data to the
* buffer.
*/
if ((ret = s_wait_gettable(cb, -1LL)) ==
LAGOPUS_RESULT_OK) {
goto check_inf;
} else {
/*
* Any errors occur while waiting.
*/
if (ret == LAGOPUS_RESULT_TIMEDOUT) {
/*
* Must not happen.
*/
lagopus_msg_fatal("Timed out must not happen here.\n");
}
}
} else {
/*
* The buffer still has data but it couldn't get all
* the data?? Must not happen??
*/
lagopus_msg_fatal("Couldn't get all the data even the data "
"available. Must not happen.\n");
}
} else {
/*
* Succeeded.
*/
ret = n_copyout;
}
} else {
ret = LAGOPUS_RESULT_NOT_OPERATIONAL;
}
}
s_unlock(cb);
} else {
s_lock(cb);
{
/*
* Repeat getting until all the required # of the data are
* got or the spcified time limit is expired.
*/
lagopus_chrono_t copy_start;
lagopus_chrono_t wait_end;
lagopus_chrono_t to = nsec;
check_to:
mbar();
if (cb->m_is_operational == true) {
WHAT_TIME_IS_IT_NOW_IN_NSEC(copy_start);
n_copyout += s_copyout(cb,
(void *)((char *)valptr +
((size_t)n_copyout * valsz)),
n_vals_max - (size_t)n_copyout,
do_incr);
if ((size_t)n_copyout < n_at_least) {
/*
* Need to repeat.
*/
if (cb->m_n_elements < 1LL) {
/*
* No data. Need to wait for someone put data to the
* buffer.
*/
if ((ret = s_wait_gettable(cb, to)) ==
LAGOPUS_RESULT_OK) {
WHAT_TIME_IS_IT_NOW_IN_NSEC(wait_end);
to -= (wait_end - copy_start);
if (to > 0LL) {
goto check_to;
}
ret = LAGOPUS_RESULT_TIMEDOUT;
}
} else {
/*
* The buffer still has data but it couldn't get all
* the data?? Must not happen??
*/
lagopus_msg_fatal("Couldn't get all the data even the data "
"available. Must not happen.\n");
}
} else {
/*
* Succeeded.
*/
ret = n_copyout;
}
} else {
ret = LAGOPUS_RESULT_NOT_OPERATIONAL;
}
}
s_unlock(cb);
}
if (n_actual_get != NULL) {
*n_actual_get = (size_t)n_copyout;
}
} else {
if (n_actual_get != NULL) {
*n_actual_get = 0LL;
}
ret = LAGOPUS_RESULT_OK;
}
} else {
if (n_actual_get != NULL) {
*n_actual_get = 0LL;
}
ret = LAGOPUS_RESULT_INVALID_ARGS;
}
return ret;
}
lagopus_chrono_t
lagopus_chrono_now(void) {
lagopus_chrono_t ret = 0;
WHAT_TIME_IS_IT_NOW_IN_NSEC(ret);
return ret;
}
static int
do_run(size_t nthds, ssize_t nputs) {
int ret = 1;
lagopus_result_t r;
size_t i;
size_t j;
char thdname[16];
lagopus_chrono_t *put_dates = NULL;
size_t n_need_watch = 0;
size_t n_valid_polls = 0;
ssize_t qsz;
test_thread_t tt;
lagopus_bbq_t bbq;
lagopus_chrono_t t_begin;
lagopus_chrono_t t_end;
lagopus_chrono_t p_begin;
lagopus_chrono_t p_t;
ssize_t n_gets = 0;
lagopus_chrono_t p_min = LLONG_MAX;
lagopus_chrono_t p_max = LLONG_MIN;
double p_sum = 0.0;
double p_sum2 = 0.0;
double p_avg;
double p_sd;
lagopus_chrono_t t_total = 0;
double t_avg;
/*
* This is only for choking clang/scan-build.
*/
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
put_dates = (lagopus_chrono_t *)
malloc(sizeof(lagopus_chrono_t) * (size_t)nputs);
if (put_dates == NULL) {
goto done;
}
if ((r = lagopus_mutex_create(&start_lock)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
if ((r = lagopus_mutex_create(&stop_lock)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
/*
* Create the qmuxer.
*/
if ((r = lagopus_qmuxer_create(&qmx)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
/*
* Then create queues.
*/
bbqs = (lagopus_bbq_t *)malloc(sizeof(lagopus_bbq_t) * nthds);
if (bbqs == NULL) {
goto done;
}
for (i = 0; i < nthds; i++) {
if ((r = lagopus_bbq_create(&(bbqs[i]), lagopus_chrono_t,
1000LL * 1000LL,
NULL)) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
n_created_bbqs++;
}
if (n_created_bbqs == 0) {
goto done;
}
/*
* Then create poll objects for the each queue.
*/
polls = (lagopus_qmuxer_poll_t *)malloc(sizeof(lagopus_qmuxer_poll_t) *
n_created_bbqs);
if (polls == NULL) {
goto done;
}
for (i = 0; i < n_created_bbqs; i++) {
if ((r = lagopus_qmuxer_poll_create(&(polls[i]),
bbqs[i],
LAGOPUS_QMUXER_POLL_READABLE)) !=
LAGOPUS_RESULT_OK) {
lagopus_perror(r);
goto done;
}
n_created_polls++;
}
if (n_created_polls == 0) {
goto done;
}
/*
* Then create threads for the each poll objects/queues.
*/
tts = (test_thread_record *)malloc(sizeof(test_thread_record) *
n_created_polls);
if (tts == NULL) {
goto done;
}
for (i = 0; i < n_created_polls; i++) {
snprintf(thdname, sizeof(thdname), "putter " PFSZS(4, u), i);
tt = &(tts[i]);
if (test_thread_create(&tt, start_lock, bbqs[i], nputs,
(const char *)thdname) != true) {
goto done;
}
n_created_thds++;
}
if (n_created_thds == 0) {
goto done;
}
/*
* Let the initiation begin.
*/
/*
* Ready, note that all the created threads do this lock.
*/
(void)lagopus_mutex_lock(&start_lock);
/*
* Steady,
*/
for (i = 0; i < n_created_thds; i++) {
tt = &(tts[i]);
if (lagopus_thread_start((lagopus_thread_t *)&tt, false) !=
LAGOPUS_RESULT_OK) {
(void)lagopus_mutex_unlock(&start_lock);
goto done;
}
}
fprintf(stdout, "Test for " PFSZ(u) " threads " PFSZ(u)
" events/thdread start.\n", n_created_thds, (size_t)nputs);
/*
* Go.
*/
(void)lagopus_mutex_unlock(&start_lock);
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
while (true) {
/*
* Like the select(2)/poll(2), initialize poll objects before
* checking events.
*/
n_need_watch = 0;
n_valid_polls = 0;
for (i = 0; i < n_created_thds; i++) {
/*
* Check if the poll has a valid queue.
*/
bbq = NULL;
if ((r = lagopus_qmuxer_poll_get_queue(&(polls[i]), &bbq)) !=
LAGOPUS_RESULT_OK) {
lagopus_perror(r);
break;
}
if (bbq != NULL) {
n_valid_polls++;
}
/*
* Reset the poll status.
*/
if ((r = lagopus_qmuxer_poll_reset(&(polls[i]))) != LAGOPUS_RESULT_OK) {
lagopus_perror(r);
break;
}
n_need_watch++;
}
/*
* If there are no valid queues, exit.
*/
if (n_valid_polls == 0) {
break;
}
/*
* Wait for an event.
*
* Note that we better set timeout, not waiting forever.
*/
r = lagopus_qmuxer_poll(&qmx, (lagopus_qmuxer_poll_t *const)polls,
n_need_watch,
100LL * 1000LL * 1000LL);
if (r > 0) {
/*
* Check which poll got an event. Actually, check all the queues
* in this sample.
*/
size_t n_actual_get = 0LL;
for (i = 0; i < n_created_thds; i++) {
if ((qsz = lagopus_bbq_size(&(bbqs[i]))) > 0) {
lagopus_msg_debug(1, "Got " PFSZS(8, u) " events from the Q"
PFSZS(03, u) ".\n",
(size_t)qsz, (size_t)i);
if ((r = lagopus_bbq_get_n(&(bbqs[i]), (void **)put_dates,
(size_t)nputs, 1LL,
lagopus_chrono_t,
1000LL * 1000LL * 1000LL,
&n_actual_get)) > 0) {
#if 1
WHAT_TIME_IS_IT_NOW_IN_NSEC(p_begin);
#endif
for (j = 0; j < n_actual_get; j++) {
/*
* In this sample, -1LL is kinda 'EOF'. Check if we got
* the EOF.
*/
if (put_dates[j] == -1LL) {
/*
* The queue is kinda 'closed'. From now on we don't
* check this queue anymore. To specify this:
*/
lagopus_msg_debug(1, "Got an EOF from the Q" PFSZS(04, u)
".\n", i);
goto nullify;
}
#if 0
WHAT_TIME_IS_IT_NOW_IN_NSEC(p_begin);
#endif
p_t = p_begin - put_dates[j];
if (p_t < p_min) {
p_min = p_t;
}
if (p_t > p_max) {
p_max = p_t;
}
p_sum += (double)p_t;
p_sum2 += ((double)p_t * (double)p_t);
n_gets++;
}
} else {
/*
* Something wrong for the queue. But we must not exit
* here. Keep on checking other queues. In order to do
* this, set NULL as the queue into the poll object for
* the queue.
*/
lagopus_perror(r);
nullify:
if ((r = lagopus_qmuxer_poll_set_queue(&(polls[i]), NULL)) ==
LAGOPUS_RESULT_OK) {
lagopus_msg_debug(1, "Q" PFSZS(04, u) " is not valid "
"anymore, ignore the queue.\n", i);
break;
} else {
/*
* There is nothing we can do now.
*/
lagopus_perror(r);
goto done;
}
}
}
}
} else if (r == LAGOPUS_RESULT_TIMEDOUT) {
lagopus_msg_debug(1, "Timedout. continue.\n");
continue;
} else {
lagopus_perror(r);
lagopus_msg_debug(1, "Break the loop due to error(s).\n");
goto done;
}
}
ret = 0;
done:
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_end);
if (is_signaled == false) {
fprintf(stdout, "Done.\n");
} else {
fprintf(stdout, "Stopped.\n");
}
fprintf(stdout, "Total # of the events:\t" PFSZS(22, u) "\n\n", n_gets);
p_avg = p_sum / (double)n_gets;
p_sd = (p_sum2 -
2.0 * p_avg * p_sum +
p_avg * p_avg * (double)n_gets) / (double)(n_gets - 1);
p_sd = sqrt(p_sd);
fprintf(stdout, "Queue stats:\n");
fprintf(stdout, "wait time min =\t" PFSZS(22, d) " nsec.\n", p_min);
fprintf(stdout, "wait time max =\t" PFSZS(22, d) " nsec.\n", p_max);
fprintf(stdout, "wait time avg =\t%25.2f nsec.\n", p_avg);
fprintf(stdout, "wait time sd =\t%25.2f.\n\n", p_sd);
t_total = t_end - t_begin;
t_avg = (double)t_total / (double)n_gets;
fprintf(stdout, "Throughput:\n");
fprintf(stdout, "total time:\t" PFSZS(22, d) " msec.\n",
(size_t)(t_total / 1000LL / 1000LL));
fprintf(stdout, "total avg:\t%25.2f nsec/event.\n", t_avg);
s_destroy_all();
free((void *)put_dates);
return ret;
}
static gallus_result_t
s_intermediate_main(const gallus_pipeline_stage_t *sptr,
size_t idx,
void *evbuf,
size_t n_evs) {
gallus_result_t ret = GALLUS_RESULT_ANY_FAILURES;
if (likely(sptr != NULL && *sptr != NULL)) {
test_stage_t ts = (test_stage_t)(*sptr);
size_t n_cur_evs = __sync_add_and_fetch(&(ts->m_n_events), 0);
if (likely(n_cur_evs < ts->m_n_data)) {
uint64_t *data = (uint64_t *)evbuf;
size_t i;
size_t j;
uint64_t sum = 0;
if (unlikely(ts->m_states[idx] == test_stage_state_initialized)) {
uint64_t cur_clock = gallus_rdtsc();
gallus_atomic_update_min(uint64_t, &(ts->m_start_clock),
0, cur_clock);
ts->m_states[idx] = test_stage_state_running;
}
for (i = 0; i < ts->m_weight; i++) {
sum = 0;
for (j = 0; j < n_evs; j++) {
sum += data[j];
}
}
(void)__sync_add_and_fetch(&(ts->m_sum), sum);
n_cur_evs = __sync_add_and_fetch(&(ts->m_n_events), n_evs);
if (unlikely(ts->m_n_data == n_cur_evs)) {
uint64_t cur_clock = gallus_rdtsc();
gallus_chrono_t end_time;
WHAT_TIME_IS_IT_NOW_IN_NSEC(end_time);
gallus_msg_debug(1, "got " PFSZ(u) " / " PFSZ(u)" events.\n",
n_cur_evs, ts->m_n_data);
gallus_atomic_update_min(uint64_t, &(ts->m_end_clock),
0, cur_clock);
gallus_atomic_update_max(gallus_chrono_t, &(ts->m_end_time),
-1, end_time);
ts->m_states[idx] = test_stage_state_done;
mbar();
if (ts->m_type == test_stage_type_egress) {
/*
* Wake the master up.
*/
(void)s_test_stage_wakeup(ts);
}
}
ret = (gallus_result_t)n_evs;
} else {
ret = s_test_stage_wait(ts);
}
} else {
ret = GALLUS_RESULT_INVALID_ARGS;
}
return ret;
}
lagopus_result_t
lagopus_module_wait_all(lagopus_chrono_t nsec) {
lagopus_result_t ret = LAGOPUS_RESULT_ANY_FAILURES;
s_lock();
{
if (s_n_modules > 0) {
lagopus_result_t first_err = LAGOPUS_RESULT_OK;
size_t i;
a_module *mptr;
/*
* Reverse order.
*/
if (nsec < 0LL) {
for (i = 0; i < s_n_modules; i++) {
mptr = &(s_modules[s_n_modules - i - 1]);
ret = s_wait_module(mptr, -1LL);
if (ret != LAGOPUS_RESULT_OK) {
lagopus_perror(ret);
lagopus_msg_error("can't wait module \"%s\".\n",
mptr->m_name);
if (first_err == LAGOPUS_RESULT_OK) {
first_err = ret;
}
}
/*
* Just carry on wait no matter what kind of errors
* occur.
*/
}
} else {
lagopus_chrono_t w_begin;
lagopus_chrono_t w_end;
lagopus_chrono_t w = nsec;
for (i = 0; i < s_n_modules; i++) {
mptr = &(s_modules[s_n_modules - i - 1]);
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_begin);
ret = s_wait_module(mptr, w);
WHAT_TIME_IS_IT_NOW_IN_NSEC(w_end);
if (ret != LAGOPUS_RESULT_OK) {
lagopus_perror(ret);
lagopus_msg_error("can't wait module \"%s\".\n",
mptr->m_name);
if (first_err == LAGOPUS_RESULT_OK) {
first_err = ret;
}
}
/*
* Just carry on wait no matter what kind of errors
* occur.
*/
w = nsec - (w_end - w_begin);
if (w < 0LL) {
w = 0LL;
}
}
}
ret = first_err;
} else {
ret = LAGOPUS_RESULT_OK;
}
}
s_unlock();
return ret;
}
int
main(int argc, const char *const argv[]) {
lagopus_result_t st = LAGOPUS_RESULT_ANY_FAILURES;
test_stage_t t = NULL;
size_t tmp;
size_t nthd = 1;
size_t c_max = 1000LL * 1000LL * 100LL;
size_t mod = 0;
lagopus_chrono_t t_begin;
lagopus_chrono_t t_end;
lagopus_chrono_t t_total;
(void)argc;
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
if (IS_VALID_STRING(argv[1]) == true) {
if (lagopus_str_parse_uint64(argv[1], &tmp) == LAGOPUS_RESULT_OK &&
tmp > 0LL) {
nthd = tmp;
}
if (IS_VALID_STRING(argv[2]) == true) {
if (lagopus_str_parse_uint64(argv[2], &tmp) == LAGOPUS_RESULT_OK &&
tmp > 0LL) {
c_max = tmp;
}
if (IS_VALID_STRING(argv[3]) == true) {
if (lagopus_str_parse_uint64(argv[3], &tmp) == LAGOPUS_RESULT_OK) {
mod = tmp;
}
}
}
}
st = lagopus_pipeline_stage_create((lagopus_pipeline_stage_t *)&t,
sizeof(*t), "a_test",
nthd,
sizeof(void *), 1024,
s_pre_pause,
s_sched,
s_setup,
/* s_fetch, */ NULL,
s_main,
/* s_throw, */ NULL,
s_shutdown,
s_finalize,
s_freeup);
if (st == LAGOPUS_RESULT_OK) {
t->m_n_max_count = c_max;
t->m_mod = mod;
st = lagopus_pipeline_stage_setup((lagopus_pipeline_stage_t *)&t);
if (st == LAGOPUS_RESULT_OK) {
st = lagopus_pipeline_stage_start((lagopus_pipeline_stage_t *)&t);
if (st == LAGOPUS_RESULT_OK) {
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_begin);
st = global_state_set(GLOBAL_STATE_STARTED);
if (st == LAGOPUS_RESULT_OK) {
sleep(1);
st = lagopus_pipeline_stage_shutdown((lagopus_pipeline_stage_t *)&t,
SHUTDOWN_GRACEFULLY);
if (st == LAGOPUS_RESULT_OK) {
st = lagopus_pipeline_stage_wait((lagopus_pipeline_stage_t *)&t,
-1LL);
}
}
}
}
}
WHAT_TIME_IS_IT_NOW_IN_NSEC(t_end);
t_total = t_end - t_begin;
if (st != LAGOPUS_RESULT_OK) {
lagopus_perror(st);
}
fprintf(stdout,
"total %f sec.\n\n"
"single thd throughput: %f Mops/s\n"
"total throuput: %f Mops/s\n\n"
"%f nsec/op\n",
(double)t_total / 1000.0 / 1000.0 / 1000.0,
(double)c_max / (double)t_total * 1000.0,
(double)c_max * (double)nthd / (double)t_total * 1000.0,
(double)t_total / (double)c_max);
return (st == LAGOPUS_RESULT_OK) ? 0 : 1;
}
