int odp_schedule_term_local(void)
{
if (sched_local.num) {
ODP_ERR("Locally pre-scheduled events exist.\n");
return -1;
}
odp_schedule_release_atomic();
sched_local_init();
return 0;
}
static int schedule_common_(void *arg)
{
thread_args_t *args = (thread_args_t *)arg;
odp_schedule_sync_t sync;
test_globals_t *globals;
queue_context *qctx;
buf_contents *bctx, *bctx_cpy;
odp_pool_t pool;
int locked;
int num;
odp_event_t ev;
odp_buffer_t buf, buf_cpy;
odp_queue_t from;
globals = args->globals;
sync = args->sync;
pool = odp_pool_lookup(MSG_POOL_NAME);
CU_ASSERT_FATAL(pool != ODP_POOL_INVALID);
if (args->num_workers > 1)
odp_barrier_wait(&globals->barrier);
while (1) {
from = ODP_QUEUE_INVALID;
num = 0;
odp_ticketlock_lock(&globals->lock);
if (globals->buf_count == 0) {
odp_ticketlock_unlock(&globals->lock);
break;
}
odp_ticketlock_unlock(&globals->lock);
if (args->enable_schd_multi) {
odp_event_t events[BURST_BUF_SIZE],
ev_cpy[BURST_BUF_SIZE];
odp_buffer_t buf_cpy[BURST_BUF_SIZE];
int j;
num = odp_schedule_multi(&from, ODP_SCHED_NO_WAIT,
events, BURST_BUF_SIZE);
CU_ASSERT(num >= 0);
CU_ASSERT(num <= BURST_BUF_SIZE);
if (num == 0)
continue;
if (sync == ODP_SCHED_SYNC_ORDERED) {
int ndx;
int ndx_max;
int rc;
ndx_max = odp_queue_lock_count(from);
CU_ASSERT_FATAL(ndx_max >= 0);
qctx = odp_queue_context(from);
for (j = 0; j < num; j++) {
bctx = odp_buffer_addr(
odp_buffer_from_event
(events[j]));
buf_cpy[j] = odp_buffer_alloc(pool);
CU_ASSERT_FATAL(buf_cpy[j] !=
ODP_BUFFER_INVALID);
bctx_cpy = odp_buffer_addr(buf_cpy[j]);
memcpy(bctx_cpy, bctx,
sizeof(buf_contents));
bctx_cpy->output_sequence =
bctx_cpy->sequence;
ev_cpy[j] =
odp_buffer_to_event(buf_cpy[j]);
}
rc = odp_queue_enq_multi(qctx->pq_handle,
ev_cpy, num);
CU_ASSERT(rc == num);
bctx = odp_buffer_addr(
odp_buffer_from_event(events[0]));
for (ndx = 0; ndx < ndx_max; ndx++) {
odp_schedule_order_lock(ndx);
CU_ASSERT(bctx->sequence ==
qctx->lock_sequence[ndx]);
qctx->lock_sequence[ndx] += num;
odp_schedule_order_unlock(ndx);
}
}
for (j = 0; j < num; j++)
odp_event_free(events[j]);
} else {
ev = odp_schedule(&from, ODP_SCHED_NO_WAIT);
if (ev == ODP_EVENT_INVALID)
continue;
buf = odp_buffer_from_event(ev);
num = 1;
if (sync == ODP_SCHED_SYNC_ORDERED) {
int ndx;
int ndx_max;
int rc;
ndx_max = odp_queue_lock_count(from);
CU_ASSERT_FATAL(ndx_max >= 0);
qctx = odp_queue_context(from);
bctx = odp_buffer_addr(buf);
buf_cpy = odp_buffer_alloc(pool);
CU_ASSERT_FATAL(buf_cpy != ODP_BUFFER_INVALID);
bctx_cpy = odp_buffer_addr(buf_cpy);
memcpy(bctx_cpy, bctx, sizeof(buf_contents));
bctx_cpy->output_sequence = bctx_cpy->sequence;
rc = odp_queue_enq(qctx->pq_handle,
odp_buffer_to_event
(buf_cpy));
CU_ASSERT(rc == 0);
for (ndx = 0; ndx < ndx_max; ndx++) {
odp_schedule_order_lock(ndx);
CU_ASSERT(bctx->sequence ==
qctx->lock_sequence[ndx]);
qctx->lock_sequence[ndx] += num;
odp_schedule_order_unlock(ndx);
}
}
odp_buffer_free(buf);
}
if (args->enable_excl_atomic) {
locked = odp_spinlock_trylock(&globals->atomic_lock);
CU_ASSERT(locked != 0);
CU_ASSERT(from != ODP_QUEUE_INVALID);
if (locked) {
int cnt;
odp_time_t time = ODP_TIME_NULL;
/* Do some work here to keep the thread busy */
for (cnt = 0; cnt < 1000; cnt++)
time = odp_time_sum(time,
odp_time_local());
odp_spinlock_unlock(&globals->atomic_lock);
}
}
if (sync == ODP_SCHED_SYNC_ATOMIC)
odp_schedule_release_atomic();
if (sync == ODP_SCHED_SYNC_ORDERED)
odp_schedule_release_ordered();
odp_ticketlock_lock(&globals->lock);
globals->buf_count -= num;
if (globals->buf_count < 0) {
odp_ticketlock_unlock(&globals->lock);
CU_FAIL_FATAL("Buffer counting failed");
}
odp_ticketlock_unlock(&globals->lock);
}
if (args->num_workers > 1)
odp_barrier_wait(&globals->barrier);
if (sync == ODP_SCHED_SYNC_ORDERED)
locked = odp_ticketlock_trylock(&globals->lock);
else
locked = 0;
if (locked && globals->buf_count_cpy > 0) {
odp_event_t ev;
odp_queue_t pq;
uint64_t seq;
uint64_t bcount = 0;
int i, j;
char name[32];
uint64_t num_bufs = args->num_bufs;
uint64_t buf_count = globals->buf_count_cpy;
for (i = 0; i < args->num_prio; i++) {
for (j = 0; j < args->num_queues; j++) {
snprintf(name, sizeof(name),
"plain_%d_%d_o", i, j);
pq = odp_queue_lookup(name);
CU_ASSERT_FATAL(pq != ODP_QUEUE_INVALID);
seq = 0;
while (1) {
ev = odp_queue_deq(pq);
if (ev == ODP_EVENT_INVALID) {
CU_ASSERT(seq == num_bufs);
break;
}
bctx = odp_buffer_addr(
odp_buffer_from_event(ev));
CU_ASSERT(bctx->sequence == seq);
seq++;
bcount++;
odp_event_free(ev);
}
}
}
CU_ASSERT(bcount == buf_count);
globals->buf_count_cpy = 0;
}
if (locked)
odp_ticketlock_unlock(&globals->lock);
/* Clear scheduler atomic / ordered context between tests */
num = exit_schedule_loop();
CU_ASSERT(num == 0);
if (num)
printf("\nDROPPED %i events\n\n", num);
return 0;
}
/*
* Schedule queues
*
* TODO: SYNC_ORDERED not implemented yet
*/
static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
unsigned int max_num, unsigned int max_deq)
{
int i, j;
int thr;
int ret;
if (sched_local.num) {
ret = copy_events(out_ev, max_num);
if (out_queue)
*out_queue = queue_handle(sched_local.qe);
return ret;
}
odp_schedule_release_atomic();
if (odp_unlikely(sched_local.pause))
return 0;
thr = odp_thread_id();
for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) {
int id;
if (sched->pri_mask[i] == 0)
continue;
id = thr & (QUEUES_PER_PRIO-1);
for (j = 0; j < QUEUES_PER_PRIO; j++, id++) {
odp_queue_t pri_q;
odp_event_t ev;
odp_buffer_t buf;
sched_cmd_t *sched_cmd;
queue_entry_t *qe;
int num;
if (id >= QUEUES_PER_PRIO)
id = 0;
if (odp_unlikely((sched->pri_mask[i] & (1 << id)) == 0))
continue;
pri_q = sched->pri_queue[i][id];
ev = odp_queue_deq(pri_q);
buf = odp_buffer_from_event(ev);
if (buf == ODP_BUFFER_INVALID)
continue;
sched_cmd = odp_buffer_addr(buf);
if (sched_cmd->cmd == SCHED_CMD_POLL_PKTIN) {
/* Poll packet input */
if (pktin_poll(sched_cmd->pe)) {
/* Stop scheduling the pktio */
pri_clr_pktio(sched_cmd->pktio,
sched_cmd->prio);
odp_buffer_free(buf);
} else {
/* Continue scheduling the pktio */
if (odp_queue_enq(pri_q, ev))
ODP_ABORT("schedule failed\n");
}
continue;
}
qe = sched_cmd->qe;
num = queue_deq_multi(qe, sched_local.buf_hdr, max_deq);
if (num < 0) {
/* Destroyed queue */
queue_destroy_finalize(qe);
continue;
}
if (num == 0) {
/* Remove empty queue from scheduling */
continue;
}
sched_local.num = num;
sched_local.index = 0;
sched_local.qe = qe;
ret = copy_events(out_ev, max_num);
if (queue_is_atomic(qe)) {
/* Hold queue during atomic access */
sched_local.pri_queue = pri_q;
sched_local.cmd_ev = ev;
} else {
/* Continue scheduling the queue */
if (odp_queue_enq(pri_q, ev))
ODP_ABORT("schedule failed\n");
}
/* Output the source queue handle */
if (out_queue)
*out_queue = queue_handle(qe);
return ret;
}
}
return 0;
}
