static odp_event_t queue_deq_wait_time(odp_queue_t queue, uint64_t ns)
{
odp_time_t wait, end;
odp_event_t ev;
wait = odp_time_local_from_ns(ns);
end = odp_time_sum(odp_time_local(), wait);
do {
ev = odp_queue_deq(queue);
if (ev != ODP_EVENT_INVALID)
return ev;
} while (odp_time_cmp(end, odp_time_local()) > 0);
return ODP_EVENT_INVALID;
}
static odp_packet_t wait_for_packet(pktio_info_t *pktio_rx,
uint32_t seq, uint64_t ns)
{
odp_time_t wait_time, end;
odp_event_t ev;
odp_packet_t pkt;
uint64_t wait;
wait = odp_schedule_wait_time(ns);
wait_time = odp_time_local_from_ns(ns);
end = odp_time_sum(odp_time_local(), wait_time);
do {
pkt = ODP_PACKET_INVALID;
if (pktio_rx->in_mode == ODP_PKTIN_MODE_RECV) {
odp_pktio_recv(pktio_rx->id, &pkt, 1);
} else {
if (pktio_rx->in_mode == ODP_PKTIN_MODE_POLL)
ev = queue_deq_wait_time(pktio_rx->inq, ns);
else
ev = odp_schedule(NULL, wait);
if (ev != ODP_EVENT_INVALID) {
if (odp_event_type(ev) == ODP_EVENT_PACKET)
pkt = odp_packet_from_event(ev);
else
odp_event_free(ev);
}
}
if (pkt != ODP_PACKET_INVALID) {
if (pktio_pkt_seq(pkt) == seq)
return pkt;
odp_packet_free(pkt);
}
} while (odp_time_cmp(end, odp_time_local()) > 0);
CU_FAIL("failed to receive transmitted packet");
return ODP_PACKET_INVALID;
}
static int schedule_common_(void *arg)
{
thread_args_t *args = (thread_args_t *)arg;
odp_schedule_sync_t sync;
test_globals_t *globals;
queue_context *qctx;
buf_contents *bctx, *bctx_cpy;
odp_pool_t pool;
int locked;
int num;
odp_event_t ev;
odp_buffer_t buf, buf_cpy;
odp_queue_t from;
globals = args->globals;
sync = args->sync;
pool = odp_pool_lookup(MSG_POOL_NAME);
CU_ASSERT_FATAL(pool != ODP_POOL_INVALID);
if (args->num_workers > 1)
odp_barrier_wait(&globals->barrier);
while (1) {
from = ODP_QUEUE_INVALID;
num = 0;
odp_ticketlock_lock(&globals->lock);
if (globals->buf_count == 0) {
odp_ticketlock_unlock(&globals->lock);
break;
}
odp_ticketlock_unlock(&globals->lock);
if (args->enable_schd_multi) {
odp_event_t events[BURST_BUF_SIZE],
ev_cpy[BURST_BUF_SIZE];
odp_buffer_t buf_cpy[BURST_BUF_SIZE];
int j;
num = odp_schedule_multi(&from, ODP_SCHED_NO_WAIT,
events, BURST_BUF_SIZE);
CU_ASSERT(num >= 0);
CU_ASSERT(num <= BURST_BUF_SIZE);
if (num == 0)
continue;
if (sync == ODP_SCHED_SYNC_ORDERED) {
int ndx;
int ndx_max;
int rc;
ndx_max = odp_queue_lock_count(from);
CU_ASSERT_FATAL(ndx_max >= 0);
qctx = odp_queue_context(from);
for (j = 0; j < num; j++) {
bctx = odp_buffer_addr(
odp_buffer_from_event
(events[j]));
buf_cpy[j] = odp_buffer_alloc(pool);
CU_ASSERT_FATAL(buf_cpy[j] !=
ODP_BUFFER_INVALID);
bctx_cpy = odp_buffer_addr(buf_cpy[j]);
memcpy(bctx_cpy, bctx,
sizeof(buf_contents));
bctx_cpy->output_sequence =
bctx_cpy->sequence;
ev_cpy[j] =
odp_buffer_to_event(buf_cpy[j]);
}
rc = odp_queue_enq_multi(qctx->pq_handle,
ev_cpy, num);
CU_ASSERT(rc == num);
bctx = odp_buffer_addr(
odp_buffer_from_event(events[0]));
for (ndx = 0; ndx < ndx_max; ndx++) {
odp_schedule_order_lock(ndx);
CU_ASSERT(bctx->sequence ==
qctx->lock_sequence[ndx]);
qctx->lock_sequence[ndx] += num;
odp_schedule_order_unlock(ndx);
}
}
for (j = 0; j < num; j++)
odp_event_free(events[j]);
} else {
ev = odp_schedule(&from, ODP_SCHED_NO_WAIT);
if (ev == ODP_EVENT_INVALID)
continue;
buf = odp_buffer_from_event(ev);
num = 1;
if (sync == ODP_SCHED_SYNC_ORDERED) {
int ndx;
int ndx_max;
int rc;
ndx_max = odp_queue_lock_count(from);
CU_ASSERT_FATAL(ndx_max >= 0);
qctx = odp_queue_context(from);
bctx = odp_buffer_addr(buf);
buf_cpy = odp_buffer_alloc(pool);
CU_ASSERT_FATAL(buf_cpy != ODP_BUFFER_INVALID);
bctx_cpy = odp_buffer_addr(buf_cpy);
memcpy(bctx_cpy, bctx, sizeof(buf_contents));
bctx_cpy->output_sequence = bctx_cpy->sequence;
rc = odp_queue_enq(qctx->pq_handle,
odp_buffer_to_event
(buf_cpy));
CU_ASSERT(rc == 0);
for (ndx = 0; ndx < ndx_max; ndx++) {
odp_schedule_order_lock(ndx);
CU_ASSERT(bctx->sequence ==
qctx->lock_sequence[ndx]);
qctx->lock_sequence[ndx] += num;
odp_schedule_order_unlock(ndx);
}
}
odp_buffer_free(buf);
}
if (args->enable_excl_atomic) {
locked = odp_spinlock_trylock(&globals->atomic_lock);
CU_ASSERT(locked != 0);
CU_ASSERT(from != ODP_QUEUE_INVALID);
if (locked) {
int cnt;
odp_time_t time = ODP_TIME_NULL;
/* Do some work here to keep the thread busy */
for (cnt = 0; cnt < 1000; cnt++)
time = odp_time_sum(time,
odp_time_local());
odp_spinlock_unlock(&globals->atomic_lock);
}
}
if (sync == ODP_SCHED_SYNC_ATOMIC)
odp_schedule_release_atomic();
if (sync == ODP_SCHED_SYNC_ORDERED)
odp_schedule_release_ordered();
odp_ticketlock_lock(&globals->lock);
globals->buf_count -= num;
if (globals->buf_count < 0) {
odp_ticketlock_unlock(&globals->lock);
CU_FAIL_FATAL("Buffer counting failed");
}
odp_ticketlock_unlock(&globals->lock);
}
if (args->num_workers > 1)
odp_barrier_wait(&globals->barrier);
if (sync == ODP_SCHED_SYNC_ORDERED)
locked = odp_ticketlock_trylock(&globals->lock);
else
locked = 0;
if (locked && globals->buf_count_cpy > 0) {
odp_event_t ev;
odp_queue_t pq;
uint64_t seq;
uint64_t bcount = 0;
int i, j;
char name[32];
uint64_t num_bufs = args->num_bufs;
uint64_t buf_count = globals->buf_count_cpy;
for (i = 0; i < args->num_prio; i++) {
for (j = 0; j < args->num_queues; j++) {
snprintf(name, sizeof(name),
"plain_%d_%d_o", i, j);
pq = odp_queue_lookup(name);
CU_ASSERT_FATAL(pq != ODP_QUEUE_INVALID);
seq = 0;
while (1) {
ev = odp_queue_deq(pq);
if (ev == ODP_EVENT_INVALID) {
CU_ASSERT(seq == num_bufs);
break;
}
bctx = odp_buffer_addr(
odp_buffer_from_event(ev));
CU_ASSERT(bctx->sequence == seq);
seq++;
bcount++;
odp_event_free(ev);
}
}
}
CU_ASSERT(bcount == buf_count);
globals->buf_count_cpy = 0;
}
if (locked)
odp_ticketlock_unlock(&globals->lock);
/* Clear scheduler atomic / ordered context between tests */
num = exit_schedule_loop();
CU_ASSERT(num == 0);
if (num)
printf("\nDROPPED %i events\n\n", num);
return 0;
}
/*
* Main packet transmit routine. Transmit packets at a fixed rate for
* specified length of time.
*/
static int run_thread_tx(void *arg)
{
test_globals_t *globals;
int thr_id;
odp_pktout_queue_t pktout;
pkt_tx_stats_t *stats;
odp_time_t cur_time, send_time_end, send_duration;
odp_time_t burst_gap_end, burst_gap;
uint32_t batch_len;
int unsent_pkts = 0;
odp_packet_t tx_packet[BATCH_LEN_MAX];
odp_time_t idle_start = ODP_TIME_NULL;
thread_args_t *targs = arg;
batch_len = targs->batch_len;
if (batch_len > BATCH_LEN_MAX)
batch_len = BATCH_LEN_MAX;
thr_id = odp_thread_id();
globals = odp_shm_addr(odp_shm_lookup("test_globals"));
stats = &globals->tx_stats[thr_id];
if (odp_pktout_queue(globals->pktio_tx, &pktout, 1) != 1)
LOG_ABORT("Failed to get output queue for thread %d\n", thr_id);
burst_gap = odp_time_local_from_ns(
ODP_TIME_SEC_IN_NS / (targs->pps / targs->batch_len));
send_duration =
odp_time_local_from_ns(targs->duration * ODP_TIME_SEC_IN_NS);
odp_barrier_wait(&globals->tx_barrier);
cur_time = odp_time_local();
send_time_end = odp_time_sum(cur_time, send_duration);
burst_gap_end = cur_time;
while (odp_time_cmp(send_time_end, cur_time) > 0) {
unsigned alloc_cnt = 0, tx_cnt;
if (odp_time_cmp(burst_gap_end, cur_time) > 0) {
cur_time = odp_time_local();
if (!odp_time_cmp(idle_start, ODP_TIME_NULL))
idle_start = cur_time;
continue;
}
if (odp_time_cmp(idle_start, ODP_TIME_NULL) > 0) {
odp_time_t diff = odp_time_diff(cur_time, idle_start);
stats->s.idle_ticks =
odp_time_sum(diff, stats->s.idle_ticks);
idle_start = ODP_TIME_NULL;
}
burst_gap_end = odp_time_sum(burst_gap_end, burst_gap);
alloc_cnt = alloc_packets(tx_packet, batch_len - unsent_pkts);
if (alloc_cnt != batch_len)
stats->s.alloc_failures++;
tx_cnt = send_packets(pktout, tx_packet, alloc_cnt);
unsent_pkts = alloc_cnt - tx_cnt;
stats->s.enq_failures += unsent_pkts;
stats->s.tx_cnt += tx_cnt;
cur_time = odp_time_local();
}
VPRINT(" %02d: TxPkts %-8" PRIu64 " EnqFail %-6" PRIu64
" AllocFail %-6" PRIu64 " Idle %" PRIu64 "ms\n",
thr_id, stats->s.tx_cnt,
stats->s.enq_failures, stats->s.alloc_failures,
odp_time_to_ns(stats->s.idle_ticks) /
(uint64_t)ODP_TIME_MSEC_IN_NS);
return 0;
}
