int odp_hisi_timer_stop(struct odp_hisi_timer *tim)
{
union odp_hisi_timer_status prev_status, status;
unsigned core_id = odp_core_id();
int ret;
ret = timer_set_config_state(tim, &prev_status);
if (ret < 0)
return -1;
__TIMER_STAT_ADD(stop, 1);
if ((prev_status.state == ODP_HISI_TIMER_RUNNING) &&
(core_id < ODP_MAX_CORE))
priv_timer[core_id].updated = 1;
if (prev_status.state == ODP_HISI_TIMER_PENDING) {
timer_del(tim, prev_status, 0);
__TIMER_STAT_ADD(pending, -1);
}
odp_mb_full();
status.state = ODP_HISI_TIMER_STOP;
status.owner = ODP_HISI_TIMER_NO_OWNER;
tim->status.u32 = status.u32;
return 0;
}
static int __odp_hisi_timer_reset(struct odp_hisi_timer *tim, uint64_t expire,
uint64_t period, unsigned tim_core,
odp_timer_cb_t fct, void *arg,
int local_is_locked)
{
union odp_hisi_timer_status prev_status, status;
int ret;
unsigned core_id = odp_core_id();
if (tim_core == (unsigned)CORE_ID_ANY) {
if (core_id < ODP_MAX_CORE) {
tim_core = odp_get_next_core(
priv_timer[core_id].prev_core,
0, 1);
priv_timer[core_id].prev_core = tim_core;
} else {
tim_core = odp_get_next_core(CORE_ID_ANY, 0, 1);
}
}
ret = timer_set_config_state(tim, &prev_status);
if (ret < 0)
return -1;
__TIMER_STAT_ADD(reset, 1);
if ((prev_status.state == ODP_HISI_TIMER_RUNNING) &&
(core_id < ODP_MAX_CORE))
priv_timer[core_id].updated = 1;
if (prev_status.state == ODP_HISI_TIMER_PENDING) {
timer_del(tim, prev_status, local_is_locked);
__TIMER_STAT_ADD(pending, -1);
}
tim->period = period;
tim->expire = expire;
tim->f = fct;
tim->arg = arg;
__TIMER_STAT_ADD(pending, 1);
timer_add(tim, tim_core, local_is_locked);
odp_mb_full();
status.state = ODP_HISI_TIMER_PENDING;
status.owner = (int16_t)tim_core;
tim->status.u32 = status.u32;
return 0;
}
static uint32_t barrier_test(per_thread_mem_t *per_thread_mem,
odp_bool_t no_barrier_test)
{
global_shared_mem_t *global_mem;
uint32_t barrier_errs, iterations, cnt, i_am_slow_thread;
uint32_t thread_num, slow_thread_num, next_slow_thread, num_threads;
uint32_t lock_owner_delay, barrier_cnt1, barrier_cnt2;
thread_num = odp_thread_id();
global_mem = per_thread_mem->global_mem;
num_threads = global_mem->g_num_threads;
iterations = BARRIER_ITERATIONS;
barrier_errs = 0;
lock_owner_delay = SLOW_BARRIER_DELAY;
for (cnt = 1; cnt < iterations; cnt++) {
/* Wait here until all of the threads reach this point */
custom_barrier_wait(&global_mem->custom_barrier1[cnt]);
barrier_cnt1 = LOAD_U32(global_mem->barrier_cnt1);
barrier_cnt2 = LOAD_U32(global_mem->barrier_cnt2);
if ((barrier_cnt1 != cnt) || (barrier_cnt2 != cnt)) {
printf("thread_num=%" PRIu32 " barrier_cnts of %" PRIu32
" %" PRIu32 " cnt=%" PRIu32 "\n",
thread_num, barrier_cnt1, barrier_cnt2, cnt);
barrier_errs++;
}
/* Wait here until all of the threads reach this point */
custom_barrier_wait(&global_mem->custom_barrier2[cnt]);
slow_thread_num = LOAD_U32(global_mem->slow_thread_num);
i_am_slow_thread = thread_num == slow_thread_num;
next_slow_thread = slow_thread_num + 1;
if (num_threads < next_slow_thread)
next_slow_thread = 1;
/*
* Now run the test, which involves having all but one thread
* immediately calling odp_barrier_wait(), and one thread wait a
* moderate amount of time and then calling odp_barrier_wait().
* The test fails if any of the first group of threads
* has not waited for the "slow" thread. The "slow" thread is
* responsible for re-initializing the barrier for next trial.
*/
if (i_am_slow_thread) {
thread_delay(per_thread_mem, lock_owner_delay);
lock_owner_delay += BASE_DELAY;
if ((LOAD_U32(global_mem->barrier_cnt1) != cnt) ||
(LOAD_U32(global_mem->barrier_cnt2) != cnt) ||
(LOAD_U32(global_mem->slow_thread_num)
!= slow_thread_num))
barrier_errs++;
}
if (no_barrier_test == 0)
odp_barrier_wait(&global_mem->test_barriers[cnt]);
STORE_U32(global_mem->barrier_cnt1, cnt + 1);
odp_mb_full();
if (i_am_slow_thread) {
STORE_U32(global_mem->slow_thread_num, next_slow_thread);
STORE_U32(global_mem->barrier_cnt2, cnt + 1);
odp_mb_full();
} else {
uint32_t cnt2 = LOAD_U32(global_mem->barrier_cnt2);
while (cnt2 != (cnt + 1)){
thread_delay(per_thread_mem, BASE_DELAY);
cnt2 = LOAD_U32(global_mem->barrier_cnt2);
}
}
}
if ((global_mem->g_verbose) && (barrier_errs != 0))
printf("\nThread %" PRIu32 " (id=%d core=%d) had %" PRIu32
" barrier_errs in %" PRIu32 " iterations\n", thread_num,
per_thread_mem->thread_id,
per_thread_mem->thread_core, barrier_errs, iterations);
return barrier_errs;
}
void odp_hisi_timer_manage(void)
{
union odp_hisi_timer_status status;
struct odp_hisi_timer *tim, *next_tim;
unsigned core_id = odp_core_id();
struct odp_hisi_timer *prev[MAX_SKIPLIST_DEPTH + 1];
uint64_t cur_time;
int i, ret;
assert(core_id < ODP_MAX_CORE);
__TIMER_STAT_ADD(manage, 1);
if (!priv_timer[core_id].pending_head.sl_next[0])
return;
cur_time = odp_get_tsc_cycles();
odp_spinlock_lock(&priv_timer[core_id].list_lock);
if ((!priv_timer[core_id].pending_head.sl_next[0]) ||
(priv_timer[core_id].pending_head.sl_next[0]->expire > cur_time))
goto done;
tim = priv_timer[core_id].pending_head.sl_next[0];
timer_get_prev_entries(cur_time, core_id, prev);
for (i = priv_timer[core_id].curr_skiplist_depth - 1; i >= 0; i--) {
priv_timer[core_id].pending_head.sl_next[i] =
prev[i]->sl_next[i];
if (!prev[i]->sl_next[i])
priv_timer[core_id].curr_skiplist_depth--;
prev[i]->sl_next[i] = NULL;
}
for (; tim; tim = next_tim) {
next_tim = tim->sl_next[0];
ret = timer_set_running_state(tim);
if (ret < 0)
continue;
odp_spinlock_unlock(&priv_timer[core_id].list_lock);
priv_timer[core_id].updated = 0;
tim->f(tim, tim->arg);
odp_spinlock_lock(&priv_timer[core_id].list_lock);
__TIMER_STAT_ADD(pending, -1);
if (priv_timer[core_id].updated == 1)
continue;
if (tim->period == 0) {
status.state = ODP_HISI_TIMER_STOP;
status.owner = ODP_HISI_TIMER_NO_OWNER;
odp_mb_full();
tim->status.u32 = status.u32;
} else {
status.state = ODP_HISI_TIMER_PENDING;
__TIMER_STAT_ADD(pending, 1);
status.owner = (int16_t)core_id;
odp_mb_full();
tim->status.u32 = status.u32;
__odp_hisi_timer_reset(tim, cur_time + tim->period,
tim->period, core_id, tim->f,
tim->arg, 1);
}
}
priv_timer[core_id].pending_head.expire =
(!priv_timer[core_id].pending_head.sl_next[0]) ? 0 :
priv_timer[core_id].pending_head.sl_next[0]->expire;
done:
odp_spinlock_unlock(&priv_timer[core_id].list_lock);
}
/**
* Packet IO worker thread accessing IO resources directly
*
* @param arg thread arguments of type 'thread_args_t *'
*/
static void *pktio_direct_recv_thread(void *arg)
{
int thr;
int pkts;
odp_packet_t pkt_tbl[MAX_PKT_BURST];
int src_idx, dst_idx;
odp_pktio_t pktio_src, pktio_dst;
thread_args_t *thr_args = arg;
stats_t *stats = thr_args->stats;
thr = odp_thread_id();
src_idx = thr_args->src_idx;
dst_idx = gbl_args->dst_port[src_idx];
pktio_src = gbl_args->pktios[src_idx];
pktio_dst = gbl_args->pktios[dst_idx];
printf("[%02i] srcif:%s dstif:%s spktio:%02" PRIu64
" dpktio:%02" PRIu64 " DIRECT RECV mode\n",
thr,
gbl_args->appl.if_names[src_idx],
gbl_args->appl.if_names[dst_idx],
odp_pktio_to_u64(pktio_src), odp_pktio_to_u64(pktio_dst));
odp_barrier_wait(&barrier);
/* Loop packets */
while (!exit_threads) {
int sent, i;
unsigned tx_drops;
pkts = odp_pktio_recv(pktio_src, pkt_tbl, MAX_PKT_BURST);
if (odp_unlikely(pkts <= 0))
continue;
if (gbl_args->appl.error_check) {
int rx_drops;
/* Drop packets with errors */
rx_drops = drop_err_pkts(pkt_tbl, pkts);
if (odp_unlikely(rx_drops)) {
stats->s.rx_drops += rx_drops;
if (pkts == rx_drops)
continue;
pkts -= rx_drops;
}
}
fill_eth_addrs(pkt_tbl, pkts, dst_idx);
sent = odp_pktio_send(pktio_dst, pkt_tbl, pkts);
sent = odp_unlikely(sent < 0) ? 0 : sent;
tx_drops = pkts - sent;
if (odp_unlikely(tx_drops)) {
stats->s.tx_drops += tx_drops;
/* Drop rejected packets */
for (i = sent; i < pkts; i++)
odp_packet_free(pkt_tbl[i]);
}
stats->s.packets += pkts;
}
/* Make sure that latest stat writes are visible to other threads */
odp_mb_full();
return NULL;
}
/**
* Packet IO worker thread using ODP queues
*
* @param arg thread arguments of type 'thread_args_t *'
*/
static void *pktio_queue_thread(void *arg)
{
odp_event_t ev_tbl[MAX_PKT_BURST];
odp_packet_t pkt_tbl[MAX_PKT_BURST];
int pkts;
int thr;
uint64_t wait;
int dst_idx;
odp_pktio_t pktio_dst;
thread_args_t *thr_args = arg;
stats_t *stats = thr_args->stats;
thr = odp_thread_id();
printf("[%02i] QUEUE mode\n", thr);
odp_barrier_wait(&barrier);
wait = odp_schedule_wait_time(ODP_TIME_MSEC_IN_NS * 100);
/* Loop packets */
while (!exit_threads) {
int sent, i;
unsigned tx_drops;
pkts = odp_schedule_multi(NULL, wait, ev_tbl, MAX_PKT_BURST);
if (pkts <= 0)
continue;
for (i = 0; i < pkts; i++)
pkt_tbl[i] = odp_packet_from_event(ev_tbl[i]);
if (gbl_args->appl.error_check) {
int rx_drops;
/* Drop packets with errors */
rx_drops = drop_err_pkts(pkt_tbl, pkts);
if (odp_unlikely(rx_drops)) {
stats->s.rx_drops += rx_drops;
if (pkts == rx_drops)
continue;
pkts -= rx_drops;
}
}
/* packets from the same queue are from the same interface */
dst_idx = lookup_dest_port(pkt_tbl[0]);
fill_eth_addrs(pkt_tbl, pkts, dst_idx);
pktio_dst = gbl_args->pktios[dst_idx];
sent = odp_pktio_send(pktio_dst, pkt_tbl, pkts);
sent = odp_unlikely(sent < 0) ? 0 : sent;
tx_drops = pkts - sent;
if (odp_unlikely(tx_drops)) {
stats->s.tx_drops += tx_drops;
/* Drop rejected packets */
for (i = sent; i < pkts; i++)
odp_packet_free(pkt_tbl[i]);
}
stats->s.packets += pkts;
}
/* Make sure that latest stat writes are visible to other threads */
odp_mb_full();
return NULL;
}
/**
* Switch worker thread
*
* @param arg Thread arguments of type 'thread_args_t *'
*/
static int run_worker(void *arg)
{
thread_args_t *thr_args = arg;
odp_packet_t pkt_tbl[MAX_PKT_BURST];
odp_pktin_queue_t pktin;
odp_pktout_queue_t pktout;
unsigned num_pktio;
unsigned pktio = 0;
uint8_t port_in;
uint8_t port_out;
int pkts;
num_pktio = thr_args->num_rx_pktio;
pktin = thr_args->rx_pktio[pktio].pktin;
port_in = thr_args->rx_pktio[pktio].port_idx;
odp_barrier_wait(&barrier);
while (!exit_threads) {
int sent;
unsigned drops;
if (num_pktio > 1) {
pktin = thr_args->rx_pktio[pktio].pktin;
port_in = thr_args->rx_pktio[pktio].port_idx;
pktio++;
if (pktio == num_pktio)
pktio = 0;
}
pkts = odp_pktin_recv(pktin, pkt_tbl, MAX_PKT_BURST);
if (odp_unlikely(pkts <= 0))
continue;
thr_args->stats[port_in]->s.rx_packets += pkts;
/* Sort packets to thread local tx buffers */
forward_packets(pkt_tbl, pkts, thr_args, port_in);
/* Empty all thread local tx buffers */
for (port_out = 0; port_out < gbl_args->appl.if_count;
port_out++) {
unsigned tx_pkts;
odp_packet_t *tx_pkt_tbl;
if (port_out == port_in ||
thr_args->tx_pktio[port_out].buf.len == 0)
continue;
tx_pkts = thr_args->tx_pktio[port_out].buf.len;
thr_args->tx_pktio[port_out].buf.len = 0;
tx_pkt_tbl = thr_args->tx_pktio[port_out].buf.pkt;
pktout = thr_args->tx_pktio[port_out].pktout;
sent = odp_pktout_send(pktout, tx_pkt_tbl, tx_pkts);
sent = odp_unlikely(sent < 0) ? 0 : sent;
thr_args->stats[port_out]->s.tx_packets += sent;
drops = tx_pkts - sent;
if (odp_unlikely(drops)) {
unsigned i;
thr_args->stats[port_out]->s.tx_drops += drops;
/* Drop rejected packets */
for (i = sent; i < tx_pkts; i++)
odp_packet_free(tx_pkt_tbl[i]);
}
}
}
/* Make sure that latest stat writes are visible to other threads */
odp_mb_full();
return 0;
}
